diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 99010d81b..93d65312d 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index 910eeef0f..66af126fa 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index bf174c245..12f5ad074 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 30 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 43efd0e16..f5feb9a8f 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 48b8db457..61036d79d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -38,14 +38,14 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: c-cpp # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/autobuild@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index fc96d5c18..04bfaae88 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -20,7 +20,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 333546437..bd2d295a4 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index 0c6e5fa9c..b3e2511ae 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -31,7 +31,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -48,6 +48,6 @@ jobs: output: 'flawfinder_results.sarif' - name: Upload analysis results to GitHub Security tab - uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: sarif_file: ${{github.workspace}}/flawfinder_results.sarif diff --git a/.github/workflows/json_view_benchmarks.yml b/.github/workflows/json_view_benchmarks.yml index 305f1ef35..d87357c42 100644 --- a/.github/workflows/json_view_benchmarks.yml +++ b/.github/workflows/json_view_benchmarks.yml @@ -54,7 +54,7 @@ jobs: timeout-minutes: 60 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 21cf6c99c..35549b4e0 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index 979f2937f..12c55a0d4 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/repository_checks.yml b/.github/workflows/repository_checks.yml index b4e45f22a..c1b104ab3 100644 --- a/.github/workflows/repository_checks.yml +++ b/.github/workflows/repository_checks.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -64,7 +64,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid] steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index dbebf3e23..14b987200 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -80,6 +80,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: sarif_file: results.sarif diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 203a1416f..8b1e6ed5e 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -65,7 +65,7 @@ jobs: # Upload SARIF file generated in previous step - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: sarif_file: semgrep.sarif if: always() diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 10d527504..41928ab20 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 066f5f68b..324c15be0 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -130,7 +130,7 @@ jobs: target: [ci_cpplint] steps: &static_analysis_ubuntu_steps - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -306,7 +306,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -461,7 +461,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -499,7 +499,7 @@ jobs: runs-on: ubuntu-26.04 steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit @@ -631,7 +631,7 @@ jobs: issues: write steps: - name: Harden Runner - uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 with: egress-policy: audit diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 2f786014c..aa48c738a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -47,7 +47,7 @@ jobs: with: persist-credentials: false - name: Set up MinGW - uses: egor-tensin/setup-mingw@41b837e47d7f85214629d255b9c4bc3fcbe9fd63 # v3.0 + uses: egor-tensin/setup-mingw@d54f5ef0ce585bd3fbb1f8a71b5b254f59dbf931 # v3.0 with: platform: ${{ matrix.architecture }} version: 12.2.0 # https://github.com/egor-tensin/setup-mingw/issues/14 @@ -172,7 +172,7 @@ jobs: - name: Install Clang run: curl -fsSL -o LLVM${{ matrix.version }}.exe https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ matrix.version }}/LLVM-${{ matrix.version }}-win64.exe ; 7z x LLVM${{ matrix.version }}.exe -y -o"C:/Program Files/LLVM" - name: Set up MinGW - uses: egor-tensin/setup-mingw@41b837e47d7f85214629d255b9c4bc3fcbe9fd63 # v3.0 + uses: egor-tensin/setup-mingw@d54f5ef0ce585bd3fbb1f8a71b5b254f59dbf931 # v3.0 with: platform: x64 version: 12.2.0 # https://github.com/egor-tensin/setup-mingw/issues/14 diff --git a/README.md b/README.md index 87c70fc96..2551df959 100644 --- a/README.md +++ b/README.md @@ -1408,9 +1408,9 @@ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR I - The class contains a port of the shortest double-to-decimal conversion of [Żmij](https://github.com/vitaut/zmij) by Victor Zverovich, including the conversion of the digits to text by Xiang JunBo and the SIMD instruction sequence of Dougall Johnson, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2025 [Victor Zverovich](https://github.com/vitaut) - The class contains a copy of [Hedley](https://nemequ.github.io/hedley/) from Evan Nemerson which is licensed as [CC0-1.0](https://creativecommons.org/publicdomain/zero/1.0/). - The class contains parts of [Google Abseil](https://github.com/abseil/abseil-cpp) which is licensed under the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). -- The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, and its digit comparison for long numbers from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors +- The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, its digit comparison for long numbers, and its parsing of eight digits at a time from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors - The view's parser (``) contains techniques and code adapted from [yyjson](https://github.com/ibireme/yyjson) by YaoYuan, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above): table-driven decoding of `\u` escapes and fixed-offset unrolled checks. -- The view's parser (``) validates non-ASCII strings with the vector UTF-8 check of [simdjson](https://github.com/simdjson/simdjson) by Daniel Lemire, Geoff Langdale, John Keiser, and contributors (its "lookup4" algorithm and tables, after J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", 2021), which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here) and the Apache 2.0 License. Copyright © 2018-2025 The simdjson authors +- The class's lexer and the view's parser (``) validate non-ASCII strings with the vector UTF-8 check of [simdjson](https://github.com/simdjson/simdjson) by Daniel Lemire, Geoff Langdale, John Keiser, and contributors (its "lookup4" algorithm and tables, after J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", 2021), which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here) and the Apache 2.0 License. Copyright © 2018-2025 The simdjson authors REUSE Software diff --git a/docs/mkdocs/docs/api/basic_json/parser_callback_t.md b/docs/mkdocs/docs/api/basic_json/parser_callback_t.md index a12634b9d..643610cb2 100644 --- a/docs/mkdocs/docs/api/basic_json/parser_callback_t.md +++ b/docs/mkdocs/docs/api/basic_json/parser_callback_t.md @@ -63,6 +63,15 @@ called: Whether the JSON value which called the function during parsing should be kept (`#!cpp true`) or not (`#!cpp false`). In the latter case, it is skipped completely, or replaced by `null` if it is the top-level value. +## Notes + +For a `parse_event_t::key` event, `parsed` holds the decoded object name, with the same guarantees as the `val` +parameter of [`json_sax::key`](../json_sax/key.md#notes): all escape sequences are decoded and `\uXXXX` escapes are +stored as UTF-8, so different spellings of the same name yield identical strings; the name is validated completely +before the callback is called, so a name with an invalid escape sequence or ill-formed UTF-8 causes a parse error and +no `key` event; no Unicode normalization or case conversion is applied; and a name that occurs several times in the +same object yields one `key` event per occurrence. + ## Examples ??? example "Example: skip an object key while parsing" diff --git a/docs/mkdocs/docs/api/json_sax/key.md b/docs/mkdocs/docs/api/json_sax/key.md index 591f590e0..14b1b52a1 100644 --- a/docs/mkdocs/docs/api/json_sax/key.md +++ b/docs/mkdocs/docs/api/json_sax/key.md @@ -19,6 +19,23 @@ Whether parsing should proceed. It is safe to move the passed object key value. +When parsing JSON text, `val` is the decoded object name, not its spelling in the input: + +- All escape sequences (`\"`, `\\`, `\/`, `\b`, `\f`, `\n`, `\r`, `\t`, `\uXXXX`, and surrogate pairs) are + decoded, and `\uXXXX` escapes are stored as UTF-8. Different spellings of the same name, such as `#!json "/"` + and `#!json "\/"`, or `#!json "\u00e9"` and `#!json "\u00E9"`, yield identical strings. An escaped `\u0000` yields + an embedded NUL byte. +- The name is scanned and validated completely before `key` is called. If it contains an invalid escape sequence, an + unpaired surrogate, a control character, or ill-formed UTF-8, the parse fails with a + [`parse_error`](../../home/exceptions.md#parse-errors), and `key` is not called for it. +- No Unicode normalization or case conversion is applied: names that differ in any byte are different strings. +- `key` is called before the following `:` is read, so a parse error may still follow for the same member. +- A name that occurs several times in the same object is passed once per occurrence. + +For the binary formats, object keys contain no escape sequences. For CBOR, MessagePack, UBJSON, and BSON, `val` holds +the bytes stored in the input, and [`sax_parse`](../basic_json/sax_parse.md) passes them unchanged, without checking them +for valid UTF-8. BON8 encodes characters itself, so an ill-formed name is a parse error and `key` is not called for it. + ## Examples ??? example @@ -45,3 +62,7 @@ It is safe to move the passed object key value. ## Version history - Added in version 3.2.0. +- The behavior described in the notes above (decoded names, validation before `key` is called, no normalization, `key` + before the `:` check, one call per duplicate, bytes of binary formats passed unchanged) is the same in all versions + from 3.2.0 on, and for the `key` event of a [parser callback](../basic_json/parser_callback_t.md) from 3.0.0 on. BSON + is only supported from version 3.4.0, and BON8 only from version 3.13.0. diff --git a/docs/mkdocs/docs/features/parsing/sax_interface.md b/docs/mkdocs/docs/features/parsing/sax_interface.md index 872105a20..f732f4b2e 100644 --- a/docs/mkdocs/docs/features/parsing/sax_interface.md +++ b/docs/mkdocs/docs/features/parsing/sax_interface.md @@ -64,6 +64,9 @@ bool parse_error(std::size_t position, const std::string& last_token, const json The return value of each function determines whether parsing should proceed. +Object keys are passed to `key` decoded and validated: escape sequences are already resolved, so different spellings of +the same name yield the same string. See [`key`](../../api/json_sax/key.md#notes) for the exact guarantees. + ## Stepping through the events The events for a small input, step by step, and what two different handlers do with them. Press *Play* or step with *Next*: diff --git a/docs/mkdocs/docs/home/license.md b/docs/mkdocs/docs/home/license.md index 714e971ed..c0bb94633 100644 --- a/docs/mkdocs/docs/home/license.md +++ b/docs/mkdocs/docs/home/license.md @@ -22,8 +22,8 @@ The class contains a port of the shortest double-to-decimal conversion of [Żmij The class contains a copy of [Hedley](https://nemequ.github.io/hedley/) from Evan Nemerson which is licensed as [CC0-1.0](https://creativecommons.org/publicdomain/zero/1.0/). -The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, and its digit comparison for long numbers from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors +The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, its digit comparison for long numbers, and its parsing of eight digits at a time from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors The view's parser (``) contains techniques and code adapted from [yyjson](https://github.com/ibireme/yyjson) by YaoYuan, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above): table-driven decoding of `\u` escapes and fixed-offset unrolled checks. -The view's parser (``) validates non-ASCII strings with the vector UTF-8 check of [simdjson](https://github.com/simdjson/simdjson) by Daniel Lemire, Geoff Langdale, John Keiser, and contributors (its "lookup4" algorithm and tables, after J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", 2021), which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here) and the Apache 2.0 License. Copyright © 2018-2025 The simdjson authors +The class's lexer and the view's parser (``) validate non-ASCII strings with the vector UTF-8 check of [simdjson](https://github.com/simdjson/simdjson) by Daniel Lemire, Geoff Langdale, John Keiser, and contributors (its "lookup4" algorithm and tables, after J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per Byte", 2021), which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here) and the Apache 2.0 License. Copyright © 2018-2025 The simdjson authors diff --git a/include/nlohmann/detail/conversions/to_chars.hpp b/include/nlohmann/detail/conversions/to_chars.hpp index a977e797b..6cfb9b8e7 100644 --- a/include/nlohmann/detail/conversions/to_chars.hpp +++ b/include/nlohmann/detail/conversions/to_chars.hpp @@ -19,6 +19,11 @@ #include // conditional +// The shortest-digit conversion of doubles (see zmij.hpp) and the writing of +// its digits, including the conversion of the digits to text by Xiang JunBo +// and the SIMD instruction sequence by Dougall Johnson, are ported from Zmij +// by Victor Zverovich (https://github.com/vitaut/zmij, MIT license). +// // SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 // digits of a double at once (unless JSON_NO_SIMD selects the portable code) #if !defined(JSON_NO_SIMD) && (defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC))) diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index dce0105de..852e1a6c8 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -44,6 +44,13 @@ // use std::from_chars or std::strtold. They are free functions so the lexer // stays focused on scanning (see lexer::convert_number()) and so that other // parsers of JSON text can convert tokens exactly like it does. +// +// The Eisel-Lemire conversion (eisel_lemire()), the digit comparison for the +// cases it cannot decide, and the eight-digits-at-a-time helpers +// (is_eight_digits(), parse_eight_digits()) are adapted from fast_float +// (https://github.com/fastfloat/fast_float, Daniel Lemire and contributors, +// available under the MIT, Apache 2.0, and Boost Software licenses; used here +// under the MIT license). The table of powers of five is in pow5_table.hpp. NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail diff --git a/include/nlohmann/detail/input/simd_scan.hpp b/include/nlohmann/detail/input/simd_scan.hpp index 8caed1814..3abb911ee 100644 --- a/include/nlohmann/detail/input/simd_scan.hpp +++ b/include/nlohmann/detail/input/simd_scan.hpp @@ -30,6 +30,13 @@ // NLOHMANN_JSON_NO_TARGET_ATTRIBUTE (for compilers that ignore the target // attribute and then reject the SSSE3 intrinsics) keeps the check portable // without disabling the SSE2 code. +// +// The vector UTF-8 check (utf8_lookup4) is adapted from simdjson +// (https://github.com/simdjson/simdjson, Daniel Lemire, Geoff Langdale, John +// Keiser, and contributors, available under the Apache 2.0 and MIT licenses; +// used here under the MIT license): the "lookup4" algorithm and tables of +// J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per +// Byte" (2021). #if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #include #define JSON_SCAN_NEON 1 diff --git a/include/nlohmann/detail/view/compare.hpp b/include/nlohmann/detail/view/compare.hpp index 8ca6058f0..96a14efbb 100644 --- a/include/nlohmann/detail/view/compare.hpp +++ b/include/nlohmann/detail/view/compare.hpp @@ -8,14 +8,22 @@ #pragma once -#include // sort, stable_sort +#include // is_sorted, sort, stable_sort +#include // array #include // size_t +#include // int64_t, uint32_t, uint64_t +#include // memcpy +#include // equal_to #include // string -#include // move, pair +#include // move, pair, swap #include // vector #include +#include #include +#include +#include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -30,17 +38,355 @@ namespace view // value, at the position of the first occurrence), and in document order if // the object type keeps an order (ordered_json), by key otherwise. +/// whether n bytes are equal; the short texts of JSON (keys, numbers, most +/// strings) are compared by two overlapping loads instead of a library call +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_equal(const char* x, const char* y, std::size_t n) noexcept +{ + const auto load8 = [](const char* p) + { + std::uint64_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + const auto load4 = [](const char* p) + { + std::uint32_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + if (n >= 8) + { + for (std::size_t i = 0; i + 8 < n; i += 8) + { + if (load8(x + i) != load8(y + i)) + { + return false; + } + } + return load8(x + n - 8) == load8(y + n - 8); + } + if (n >= 4) + { + return load4(x) == load4(y) && load4(x + n - 4) == load4(y + n - 4); + } + for (std::size_t i = 0; i < n; ++i) + { + if (x[i] != y[i]) + { + return false; + } + } + return true; +} + +/// whether the n1 bytes at x are before the n2 bytes at y (memcmp() order, then the shorter first); +/// the first difference is found eight bytes at a time +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_less(const char* x, std::size_t n1, const char* y, std::size_t n2) noexcept +{ + const std::size_t n = n1 < n2 ? n1 : n2; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t u = read_eight_bytes(x + i); + const std::uint64_t v = read_eight_bytes(y + i); + if (u != v) + { + const auto shift = static_cast(count_trailing_zeros(u ^ v)) & ~7u; // (the first byte that differs) + return ((u >> shift) & 0xFFu) < ((v >> shift) & 0xFFu); + } + } + for (; i < n; ++i) + { + if (x[i] != y[i]) + { + return static_cast(x[i]) < static_cast(y[i]); + } + } + return n1 < n2; +} + +/// whether the key a is before the key b +template +NLOHMANN_VIEW_ALWAYS_INLINE bool key_less(const Key& a, const Key& b) noexcept +{ + return bytes_less(a.data(), a.size(), b.data(), b.size()); +} + +/// sorts the indices order[0..n) of the members m by key; equal keys stay in +/// document order (small objects with an insertion sort, which needs no memory) +template +void sort_by_key(std::uint32_t* order, std::size_t n, const Entry* m) +{ + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + const std::uint32_t x = order[i]; + std::size_t j = i; + for (; j > 0 && key_less(m[x].first, m[order[j - 1]].first); --j) + { + order[j] = order[j - 1]; + } + order[j] = x; + } + } + else + { + std::stable_sort(order, order + n, [m](std::uint32_t x, std::uint32_t y) + { + return key_less(m[x].first, m[y].first); + }); + } +} + +/// The members of an object as parse() keeps them: one per key, the last +/// value at the position of the first occurrence, in the order of the +/// comparison (sorted by key, or, if the object type keeps the order, the order +/// of first occurrence). The members m[0..n) are in document order; this +/// appends to out the indices of the member of each key that has its value +/// (its last one), in the order of the comparison. Most objects have unique +/// keys. scratch is working memory, kept by the caller so that its capacity +/// is reused. +template +void resolve_indices(const Entry* m, std::size_t n, bool ordered, std::vector& out, std::vector& scratch) +{ + const std::size_t start = out.size(); + out.resize(start + n); + std::uint32_t* const order = out.data() + start; + sort_by_key(order, n, m); + if (!ordered) + { + // the last of each run of equal keys + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + order[w++] = order[last]; + i = last + 1; + } + out.resize(start + w); + return; + } + // (document order) the value of the last occurrence at the first occurrence of each key + constexpr std::uint32_t none = 0xFFFFFFFFu; + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + if (last != i) + { + if (!any) + { + scratch.assign(n, none); + for (std::size_t k = 0; k < i; ++k) + { + scratch[order[k]] = order[k]; + } + any = true; + } + scratch[order[i]] = order[last]; // (stable: order[i] is the first occurrence) + } + else if (any) + { + scratch[order[i]] = order[i]; + } + i = last + 1; + } + if (!any) + { + // unique keys: document order + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + return; + } + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (scratch[i] != none) + { + order[w++] = scratch[i]; + } + } + out.resize(start + w); +} + +/// whether the keys of n members are the same, in the same order +template +bool same_keys(const EntryA* a, const EntryB* b, std::size_t n) +{ + for (std::size_t i = 0; i < n; ++i) + { + if (!(a[i].first == b[i].first)) + { + return false; + } + } + return true; +} + +/// The members of an object as parse() keeps them (one per key: the last +/// value at the position of the first occurrence), sorted by key or, if the +/// object type keeps the order, in the order of first occurrence. The +/// members are given in document order, at the end of m (from the position +/// base on); most objects have unique keys, in which case they are only sorted +/// (for small objects with an insertion sort, which keeps equal keys in order +/// and needs no memory). scratch is working memory, kept by the caller. +template +void resolve_members(std::vector>& m, std::size_t base, bool ordered, std::vector& scratch) +{ + using entry = std::pair; + const std::size_t n = m.size() - base; + entry* const first = m.data() + base; + if (ordered) + { + // (document order) later values of a key go to its first occurrence; + // scratch holds the order by key, then whether a member is dropped + constexpr std::uint32_t dropped = 1; + scratch.assign(2 * n, 0); + std::uint32_t* const order = scratch.data(); + std::uint32_t* const drop = order + n; + sort_by_key(order, n, first); + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[order[last + 1]].first == first[order[i]].first) + { + ++last; + } + if (last != i) + { + first[order[i]].second = first[order[last]].second; + for (std::size_t k = i + 1; k <= last; ++k) + { + drop[order[k]] = dropped; + } + any = true; + } + i = last + 1; + } + if (any) + { + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (drop[i] != dropped) + { + if (w != i) + { + first[w] = std::move(first[i]); + } + ++w; + } + } + m.erase(m.begin() + static_cast(base + w), m.end()); + } + return; + } + // sorted by key; equal keys stay in document order + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + entry x = std::move(first[i]); + std::size_t j = i; + for (; j > 0 && key_less(x.first, first[j - 1].first); --j) + { + first[j] = std::move(first[j - 1]); + } + first[j] = std::move(x); + } + } + else + { + std::stable_sort(first, first + n, [](const entry & x, const entry & y) + { + return key_less(x.first, y.first); + }); + } + // the first key of a run with the last value + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[last + 1].first == first[i].first) + { + ++last; + } + if (w != i || last != i) + { + first[w] = entry(first[i].first, std::move(first[last].second)); + } + ++w; + i = last + 1; + } + m.erase(m.begin() + static_cast(base + w), m.end()); +} + +/// room for k more entries, growing geometrically (reserve(size() + k) alone would +/// reallocate for each object nested in another) +template +void reserve_more(std::vector& v, std::size_t k) +{ + const std::size_t needed = v.size() + k; + if (needed > v.capacity()) + { + v.reserve((std::max)(needed, 2 * v.capacity())); + } +} + /// one side of a comparison: a view template class view_side { public: using string_view_t = typename View::string_view_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using nav = navigation; + + /// the position in the elements of an array + struct cursor + { + const node* pos = nullptr; + }; + + view_side() noexcept + : m_view() + {} explicit view_side(const View& v) noexcept : m_view(v) {} + /// the document and the node of the view (nullptr: a discarded view) + const document_data& document() const noexcept + { + return *m_view.m_doc; + } + + const node* value_node() const noexcept + { + return m_view.m_node; + } + + static constexpr bool is_editable = View::is_editable; + value_t type() const noexcept { return m_view.type(); @@ -56,6 +402,32 @@ class view_side return m_view.get_string(); } + bool boolean() const noexcept + { + return (m_view.m_node->flags & node_flags::is_true) != 0; + } + + number_integer_t integer() const noexcept + { + return static_cast(static_cast(integer_bits(*m_view.m_node))); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return static_cast(integer_bits(*m_view.m_node)); + } + + number_float_t floating() const + { + return float_value(*m_view.m_doc, *m_view.m_node); + } + + /// the token of a float + string_view_t float_token() const noexcept + { + return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); + } + /// a number, boolean, or null as a basic_json value (no allocation) BasicJsonType scalar() const { @@ -80,60 +452,47 @@ class view_side } } - void elements(std::vector& out) const + /// start at the first element of this array + void enter(cursor& c) const noexcept { - out.reserve(m_view.size()); - for (const View e : m_view) + c.pos = nav::first(*m_view.m_doc, m_view.m_node); + } + + /// the element at c (and advance) + view_side take(cursor& c) const noexcept + { + const node* const v = nav::value(c.pos); + c.pos = document_data::after(c.pos); + return view_side(View(m_view.m_doc, v)); + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const noexcept + { + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = 0; i < n; ++i) { - out.emplace_back(e); + out[i] = string_view_t(m_view.m_doc->str(*pos), pos->len); + pos = document_data::after(pos + 1); } } - /// the members as parse() keeps them: one per key, the last value at the - /// position of the first occurrence; in that order, or sorted by key - void members(std::vector>& out, bool ordered) const + /// append the members to out, in document order; whether their keys ascend + /// (then they are unique, and in the order of the comparison of a std::map) + bool members(std::vector>& out, bool /*ordered*/) const { - struct member + const std::size_t base = out.size(); + reserve_more(out, m_view.m_node->len); + bool sorted = true; + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = m_view.m_node->len; i != 0; --i) { - string_view_t key; - View value; - std::size_t position; - }; - std::vector all; - all.reserve(m_view.size()); - std::size_t position = 0; - for (auto it = m_view.begin(); it != m_view.end(); ++it) - { - all.push_back(member{it.key(), it.value(), position++}); - } - std::stable_sort(all.begin(), all.end(), [](const member & a, const member & b) - { - return a.key < b.key; - }); - std::vector unique; - unique.reserve(all.size()); - for (std::size_t i = 0; i < all.size();) - { - std::size_t last = i; - while (last + 1 < all.size() && all[last + 1].key == all[i].key) - { - ++last; - } - unique.push_back(member{all[i].key, all[last].value, all[i].position}); - i = last + 1; - } - if (ordered) - { - std::sort(unique.begin(), unique.end(), [](const member & a, const member & b) - { - return a.position < b.position; - }); - } - out.reserve(unique.size()); - for (const member& m : unique) - { - out.emplace_back(m.key, view_side(m.value)); + const string_view_t key(m_view.m_doc->str(*pos), pos->len); + sorted = sorted && (out.size() == base || key_less(out.back().first, key)); + out.emplace_back(key, view_side(View(m_view.m_doc, nav::value(pos + 1)))); + pos = document_data::after(pos + 1); } + return sorted; } private: @@ -146,6 +505,17 @@ class json_side { public: using string_view_t = StringView; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + /// the position in the elements of an array + struct cursor + { + typename BasicJsonType::const_iterator it{}; + }; + + json_side() noexcept = default; explicit json_side(const BasicJsonType& j) noexcept : m_json(&j) @@ -167,45 +537,428 @@ class json_side return string_view_t(s.data(), s.size()); } + bool boolean() const noexcept + { + return *m_json->template get_ptr(); + } + + number_integer_t integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_float_t floating() const noexcept + { + return *m_json->template get_ptr(); + } + BasicJsonType scalar() const { return *m_json; } - void elements(std::vector& out) const + void enter(cursor& c) const { - out.reserve(m_json->size()); - for (const auto& e : *m_json) + c.it = m_json->cbegin(); + } + + json_side take(cursor& c) const + { + const json_side r(*c.it); + ++c.it; + return r; + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const + { + auto it = m_json->cbegin(); + for (std::size_t i = 0; i < n; ++i, ++it) { - out.emplace_back(e); + out[i] = string_view_t(it.key().data(), it.key().size()); } } - void members(std::vector>& out, bool ordered) const + /// append the members to out, in the order of the comparison (the keys of a + /// basic_json are unique); true + bool members(std::vector>& out, bool ordered) const { - out.reserve(m_json->size()); + const std::size_t base = out.size(); + reserve_more(out, m_json->size()); for (auto it = m_json->cbegin(); it != m_json->cend(); ++it) { out.emplace_back(string_view_t(it.key().data(), it.key().size()), json_side(it.value())); } if (!ordered) { - std::sort(out.begin(), out.end(), [](const std::pair& a, const std::pair& b) + const auto less = [](const std::pair& a, const std::pair& b) { - return a.first < b.first; - }); + return key_less(a.first, b.first); + }; + const auto first = out.begin() + static_cast(base); + if (!std::is_sorted(first, out.end(), less)) + { + std::sort(first, out.end(), less); + } + } + return true; + } + + private: + const BasicJsonType* m_json = nullptr; +}; + +/*! +@brief the values at two nodes, found equal by walking both documents in the +same order + +Two values are equal if they have the same type, the same number token or +string, the same elements, and the same members with the same keys in the same +order: a value that matches this way is equal under the rules above whatever +the object type and duplicate keys (the last value of a key then wins on +both sides, and their order of first occurrence is the same). The walk does +not find every equal pair (numbers spelled differently, members in another +order, duplicate keys with other values), only the ones that are the same +text; it stops at the first difference and then says nothing. Containers +nested deeper than the budget are not walked. + +The walk goes through the document in its order, the comparison that follows +goes through the members of an object by key. So that the walk never reaches +a difference later than the comparison would, and never costs more than it, +it enters an object only if the two orders agree (the keys ascend, or the +object type keeps the order) or if the walk is cheaper than preparing the +comparison of the object (all values are scalars). +*/ +/// the containers that lockstep found different, from the outermost to the +/// innermost: the comparison that goes on from there does not walk them again +struct lockstep_path +{ + /// the nesting depth up to which lockstep walks (the recursion is bounded) + static constexpr unsigned budget() noexcept + { + return 48; + } + + /// a pair of containers that differ; for arrays, the elements where the walk stopped + /// (the ones before are equal) + struct entry + { + const node* a; + const node* b; + const node* element_a; + const node* element_b; + std::uint32_t index; + }; + + /// the number of entries that a path can hold (a container deeper than the + /// budget of the walk is not entered) + static constexpr std::size_t levels() noexcept + { + return 50; + } + + explicit lockstep_path(entry* storage) noexcept + : nodes(storage) + {} + + entry* nodes; ///< levels() entries, written before they are read + unsigned length = 0; ///< pairs recorded by the last walk + unsigned next = 0; ///< the pair that the comparison enters next + const entry* resume = nullptr; ///< the entry of the pair that was just looked at, if it is in the path +}; + +template +class lockstep +{ + public: + lockstep(const document_data& a, const document_data& b, lockstep_path& path, bool ordered) noexcept + : m_a(a), m_b(b), m_path(path), m_ordered(ordered) + {} + + /// whether the values at a and b are the same (the nodes of the values, + /// not entries of a sequence) + bool same(const node* a, const node* b, unsigned budget) noexcept + { + if (a->kind != b->kind) + { + return false; + } + switch (static_cast(a->kind)) + { + case value_t::null: + return true; + case value_t::boolean: + return ((a->flags ^ b->flags) & node_flags::is_true) == 0; + case value_t::string: + return a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_float: // (the token, and the digit layout that the value is read with: they only differ in a damaged image) + return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_integer: + case value_t::number_unsigned: + return integer_bits(*a) == integer_bits(*b); + case value_t::array: + if (budget == 0 || a->len != b->len) + { + return false; + } + { + lockstep_path::entry stop{a, b, nullptr, nullptr, 0}; + if (elements(a, b, budget - 1, stop)) + { + return true; + } + record(stop, budget); + return false; + } + case value_t::object: + if (budget == 0 || a->len != b->len) + { + return false; + } + if (!visits_in_order(a) || !members(a, b, budget - 1)) + { + record(lockstep_path::entry{a, b, nullptr, nullptr, 0}, budget); + return false; + } + return true; + case value_t::binary: // LCOV_EXCL_LINE (not in a document) + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE } } private: - const BasicJsonType* m_json; + using nav_a = navigation; + using nav_b = navigation; + + /// note the containers a and b (at the nesting level that budget stands + /// for) as different + NLOHMANN_VIEW_NOINLINE void record(const lockstep_path::entry& stop, unsigned budget) const noexcept + { + const unsigned level = lockstep_path::budget() - budget; + m_path.nodes[level] = stop; + if (m_path.length == 0) + { + m_path.length = level + 1; // (the innermost container is the first to be left) + } + } + + /// whether the object @a a of a document that cannot be edited has scalar values only + static bool scalar_values_only(const node* a, std::false_type /*editable*/) noexcept + { + return a->next == 1 + (2 * static_cast(a->len)); + } + + /// (the nodes of an editable document can be elsewhere: not decided) + static bool scalar_values_only(const node* /*a*/, std::true_type /*editable*/) noexcept + { + return false; + } + + /// Whether the comparison that follows visits the members of the object a in the + /// order of the document: if the object type keeps the order, if the keys + /// ascend (it sorts by key), or if all values are scalars (then the walk costs + /// less than collecting and sorting the members, whichever member differs). + /// Otherwise a difference could be found by the walk later than by the + /// comparison, which is not worth the walk. + bool visits_in_order(const node* a) const noexcept + { + if (m_ordered || a->len < 2) + { + return true; + } + if (scalar_values_only(a, std::integral_constant {})) + { + return true; + } + const node* k = nav_a::first(m_a, a); + string_ref previous(m_a.str(*k), k->len); + for (std::uint32_t i = a->len - 1; i != 0; --i) + { + k = document_data::after(k + 1); + const string_ref key(m_a.str(*k), k->len); + if (!key_less(previous, key)) + { + return false; + } + previous = key; + } + return true; + } + + bool elements(const node* a, const node* b, unsigned budget, lockstep_path::entry& stop) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (!same(nav_a::value(pa), nav_b::value(pb), budget)) + { + stop.element_a = pa; + stop.element_b = pb; + stop.index = a->len - i; + return false; + } + pa = document_data::after(pa); + pb = document_data::after(pb); + } + return true; + } + + bool members(const node* a, const node* b, unsigned budget) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (pa->len != pb->len || !bytes_equal(m_a.str(*pa), m_b.str(*pb), pa->len)) + { + return false; + } + if (!same(nav_a::value(pa + 1), nav_b::value(pb + 1), budget)) + { + return false; + } + pa = document_data::after(pa + 1); + pb = document_data::after(pb + 1); + } + return true; + } + + const document_data& m_a; + const document_data& m_b; + lockstep_path& m_path; + bool m_ordered; ///< the object type keeps the order of the members }; +/// whether two views certainly have equal values (see lockstep); if not, the +/// path says where they differ +template +bool certainly_equal(const view_side& a, const view_side& b, lockstep_path& path, bool ordered) noexcept +{ + const node* const na = a.value_node(); + const node* const nb = b.value_node(); + if (na == nullptr || nb == nullptr) + { + return false; + } + if (na == nb) + { + return true; // (the same node of the same document: documents have distinct nodes) + } + path.resume = nullptr; + if (path.next < path.length && path.nodes[path.next].a == na && path.nodes[path.next].b == nb) + { + path.resume = &path.nodes[path.next]; + ++path.next; // (the walk found these different already) + return false; + } + path.length = 0; + const bool equal = lockstep::is_editable, view_side::is_editable>(a.document(), b.document(), path, ordered).same(na, nb, lockstep_path::budget()); + path.next = path.length != 0 ? 1 : 0; // (this pair is the first in the path) + path.resume = path.length != 0 ? &path.nodes[0] : nullptr; + return equal; +} + +/// (a view and a basic_json value: no walk) +template +bool certainly_equal(const A& /*a*/, const B& /*b*/, lockstep_path& /*path*/, bool /*ordered*/) noexcept +{ + return false; +} + +/// (arrays of two views) the elements where the walk stopped are the next pair: the ones before are equal +template +void resume_array(const lockstep_path& path, const view_side& a, const view_side& b, + typename view_side::cursor& cursor_a, typename view_side::cursor& cursor_b, std::size_t& next) noexcept +{ + if (path.resume != nullptr && path.resume->element_a != nullptr && path.resume->a == a.value_node() && path.resume->b == b.value_node()) + { + cursor_a.pos = path.resume->element_a; + cursor_b.pos = path.resume->element_b; + next = path.resume->index; + } +} + +template +void resume_array(const lockstep_path& /*path*/, const A& /*a*/, const B& /*b*/, CursorA& /*cursor_a*/, CursorB& /*cursor_b*/, std::size_t& /*next*/) noexcept +{} + +/// whether two floats are equal (the tokens of two views: equal text and digit layout is an equal value; they +/// only differ in a damaged image) +template +bool floats_equal(const view_side& a, const view_side& b) +{ + return (a.value_node()->extra == b.value_node()->extra && a.float_token() == b.float_token()) || std::equal_to()(a.floating(), b.floating()); +} + +template +bool floats_equal(const A& a, const B& b) +{ + return std::equal_to()(a.floating(), b.floating()); +} + +/// Whether two objects of a type that keeps the order are different because of the keys of +/// their first members, found without the preparation of the comparison: the comparison goes +/// through the members in the order of the first occurrence of their keys, so if the first +/// members have the same distinct keys and then one pair of keys differs, where neither is +/// one of the keys before it, the objects are different. (The keys that are the same as an +/// earlier key are the duplicates that the comparison drops, so those are not decided here.) +template +bool keys_differ_early(const SideA& a, const SideB& b) +{ + using string_view_t = typename SideA::string_view_t; + constexpr std::size_t limit = 4; + const std::size_t n = (std::min)((std::min)(a.size(), b.size()), limit); + std::array ka{}; + std::array kb{}; + a.first_keys(ka.data(), n); + b.first_keys(kb.data(), n); + for (std::size_t i = 0; i < n; ++i) + { + if (ka[i] != kb[i]) + { + for (std::size_t k = 0; k < i; ++k) + { + if (ka[i] == ka[k] || kb[i] == ka[k]) + { + return false; // (a duplicate: not decided) + } + } + return true; + } + } + return false; +} + +/// keys_differ_early() for a type that keeps the order of the members +template +bool keys_differ_early(const SideA& a, const SideB& b, std::true_type /*ordered*/) +{ + return keys_differ_early(a, b); +} + +/// (a type that sorts the members by key: not decided early) +template +bool keys_differ_early(const SideA& /*a*/, const SideB& /*b*/, std::false_type /*ordered*/) noexcept +{ + return false; +} + inline bool is_number(value_t t) noexcept { return t == value_t::number_integer || t == value_t::number_unsigned || t == value_t::number_float; } +/// the index_a/index_b of a frame of equal() whose members are compared in document order +constexpr std::size_t document_order = static_cast(-1); + /// whether two sides are equal; iterative, so that the nesting depth is /// limited by memory only template @@ -216,20 +969,52 @@ bool equal(const A& a0, const B& b0) struct frame { - std::vector elements_a{}; - std::vector elements_b{}; - std::vector> members_a{}; - std::vector> members_b{}; + A array_a{}; + B array_b{}; + typename A::cursor cursor_a{}; + typename B::cursor cursor_b{}; + std::size_t base_a = 0; ///< the members of an object: from this position in members_a + std::size_t base_b = 0; + std::size_t index_a = document_order; ///< their order in the comparison: from this position in indices (document_order: as they are) + std::size_t index_b = document_order; + std::size_t index_mark = 0; ///< the size of indices before this frame bool object = false; std::size_t next = 0; + std::size_t count = 0; }; std::vector stack; + std::vector indices; // the order of the members of the open objects, where it is not their order in the document + std::vector scratch; // working memory of resolve_indices() and resolve_members() + // the members of the open objects (the ones of an object follow those of the objects around it) + std::vector> members_a; + std::vector> members_b; + std::array path_storage; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + lockstep_path path(path_storage.data()); + { + // (values that differ in type or in the size of an array need no walk) + const value_t first = a0.type(); + const value_t second = b0.type(); + if (first != second && !(is_number(first) && is_number(second)) && first != value_t::discarded && second != value_t::discarded) + { + return false; + } + if (first == value_t::array && second == value_t::array && a0.size() != b0.size()) + { + return false; + } + } A a = a0; B b = b0; + // (pairs of views that are certainly equal need no further look) + if (certainly_equal(a, b, path, ordered)) + { + return true; + } for (;;) { const value_t ta = a.type(); const value_t tb = b.type(); + const bool numbers = is_number(ta) && is_number(tb); if (ta == value_t::discarded || tb == value_t::discarded) { // basic_json decides (JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON) @@ -240,7 +1025,7 @@ bool equal(const A& a0, const B& b0) } else { - if (ta != tb && !(is_number(ta) && is_number(tb))) // numbers of different types can be equal + if (ta != tb && !numbers) // numbers of different types can be equal { return false; } @@ -257,31 +1042,98 @@ bool equal(const A& a0, const B& b0) { return false; } - frame f; + if (ta == value_t::object && keys_differ_early(a, b, std::integral_constant::value> {})) + { + return false; + } + stack.emplace_back(); + frame& f = stack.back(); f.object = ta == value_t::object; if (f.object) { - a.members(f.members_a, ordered); - b.members(f.members_b, ordered); - if (f.members_a.size() != f.members_b.size()) + f.base_a = members_a.size(); + f.base_b = members_b.size(); + f.index_mark = indices.size(); + const bool sorted_a = a.members(members_a, ordered); + const bool sorted_b = b.members(members_b, ordered); + const std::size_t size_a = members_a.size() - f.base_a; + const std::size_t size_b = members_b.size() - f.base_b; + std::size_t count_a = size_a; + std::size_t count_b = size_b; + if (!sorted_a && !sorted_b && size_a == size_b && same_keys(members_a.data() + f.base_a, members_b.data() + f.base_b, size_a)) + { + // the same keys in the same order: one order of the comparison for both + f.index_a = indices.size(); + f.index_b = f.index_a; + resolve_indices(members_a.data() + f.base_a, size_a, ordered, indices, scratch); + count_a = indices.size() - f.index_a; + count_b = count_a; + } + else + { + if (!sorted_a) + { + resolve_members(members_a, f.base_a, ordered, scratch); + count_a = members_a.size() - f.base_a; + } + if (!sorted_b) + { + resolve_members(members_b, f.base_b, ordered, scratch); + count_b = members_b.size() - f.base_b; + } + } + f.count = count_a; + if (count_a != count_b) { return false; } } else { - a.elements(f.elements_a); - b.elements(f.elements_b); + f.array_a = a; + f.array_b = b; + a.enter(f.cursor_a); + b.enter(f.cursor_b); + f.count = a.size(); + resume_array(path, a, b, f.cursor_a, f.cursor_b, f.next); } - stack.push_back(std::move(f)); } - else if (a.scalar() != b.scalar()) // numbers (also of different types), null, boolean + else if (ta == value_t::boolean) + { + if (a.boolean() != b.boolean()) + { + return false; + } + } + else if (numbers && ta == tb) + { + // numbers of one type: as basic_json compares them + bool differ = false; + if (ta == value_t::number_integer) + { + differ = a.integer() != b.integer(); + } + else if (ta == value_t::number_unsigned) + { + differ = a.unsigned_integer() != b.unsigned_integer(); + } + else + { + differ = !floats_equal(a, b); + } + if (differ) + { + return false; + } + } + else if (numbers && !(a.scalar() == b.scalar())) // numbers of different types { return false; } + // (null equals null) } - // the next pair of values + // the next pair of values that is not certainly equal for (;;) { if (stack.empty()) @@ -289,28 +1141,38 @@ bool equal(const A& a0, const B& b0) return true; } frame& f = stack.back(); - const std::size_t count = f.object ? f.members_a.size() : f.elements_a.size(); - if (f.next == count) + if (f.next == f.count) { + if (f.object) + { + members_a.erase(members_a.begin() + static_cast(f.base_a), members_a.end()); + members_b.erase(members_b.begin() + static_cast(f.base_b), members_b.end()); + indices.resize(f.index_mark); + } stack.pop_back(); // (f dangles now, and is bound again after continue) continue; } if (f.object) { - if (!(f.members_a[f.next].first == f.members_b[f.next].first)) + const auto& member_a = members_a[f.base_a + (f.index_a == document_order ? f.next : indices[f.index_a + f.next])]; + const auto& member_b = members_b[f.base_b + (f.index_b == document_order ? f.next : indices[f.index_b + f.next])]; + if (!(member_a.first == member_b.first)) { return false; } - a = f.members_a[f.next].second; - b = f.members_b[f.next].second; + a = member_a.second; + b = member_b.second; } else { - a = f.elements_a[f.next]; - b = f.elements_b[f.next]; + a = f.array_a.take(f.cursor_a); + b = f.array_b.take(f.cursor_b); } ++f.next; - break; + if (!certainly_equal(a, b, path, ordered)) + { + break; + } } } } diff --git a/include/nlohmann/detail/view/edit.hpp b/include/nlohmann/detail/view/edit.hpp index f47bf5985..c542c1141 100644 --- a/include/nlohmann/detail/view/edit.hpp +++ b/include/nlohmann/detail/view/edit.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -46,10 +47,9 @@ struct first_true : std::integral_constant < int, 1 + firs /// (type_error.316 with the same message), so that an editable document /// only holds valid UTF-8: the error is at the first byte that no /// well-formed sequence can continue with (Unicode, Table 3-7). -inline void check_utf8(const char* s, std::size_t n) +NLOHMANN_VIEW_ALWAYS_INLINE void check_utf8_bytewise(const unsigned char* p, std::size_t n, std::size_t i) { - const auto* const p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - for (std::size_t i = 0; i < n;) + while (i < n) { const unsigned char c = p[i]; if (c < 0x80) @@ -79,6 +79,43 @@ inline void check_utf8(const char* s, std::size_t n) } } +/// The long-string path of check_utf8(): skips well-formed runs with the +/// parser's string scan (which stops at a quote, a backslash, or a control +/// character, all legal here, and at ill-formed UTF-8); the byte-wise check +/// runs only from where the scan found ill-formed UTF-8, to report it. +NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std::size_t n) +{ + const unsigned char* const e = p + n; + const unsigned char* q = p; + for (;;) + { + q = scan_string_run(q, e); + if (q == e) + { + return; + } + if (*q >= 0x80) + { + check_utf8_bytewise(p, n, static_cast(q - p)); + return; // not reached: the scan stopped at ill-formed UTF-8 + } + ++q; // a quote, backslash, or control character + } +} + +inline void check_utf8(const char* s, std::size_t n) +{ + const auto* const p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + if (NLOHMANN_VIEW_UNLIKELY(n >= 32)) + { + check_utf8_scan(p, n); + return; + } + // short strings: the vector scan costs more than it saves; (the loop is + // inlined with a constant start, so this path is the plain byte-wise check) + check_utf8_bytewise(p, n, 0); +} + /*! @brief the edits of an editable basic_json_document diff --git a/include/nlohmann/detail/view/image.hpp b/include/nlohmann/detail/view/image.hpp index 545e04a6a..9e57082de 100644 --- a/include/nlohmann/detail/view/image.hpp +++ b/include/nlohmann/detail/view/image.hpp @@ -258,6 +258,81 @@ struct number_token bool is_float; ///< a fraction or an exponent }; +/// advance over ASCII digits, eight bytes at a time +inline const unsigned char* digit_run(const unsigned char* p, const unsigned char* e) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + while (e - p >= 8) + { + // (the digits become 0..9; a byte above 9, or with bit 7 set, is no digit. A carry + // out of a byte with bit 7 set only affects the bytes above, after the first one + // that count_trailing_zeros finds.) + const std::uint64_t x = read_eight_bytes(p) ^ (ones * 0x30u); + const std::uint64_t not_digit = ((x + (ones * 0x76u)) | x) & (ones * 0x80u); + if (not_digit != 0) + { + return p + (static_cast(count_trailing_zeros(not_digit)) / 8); + } + p += 8; + } + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} + +/// whether the n bytes at p are ASCII digits (n > 0) +inline bool all_digits(const unsigned char* p, std::size_t n) noexcept +{ + const auto digits = [](const unsigned char* q) + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + const std::uint64_t x = read_eight_bytes(q) ^ (ones * 0x30u); + return (((x + (ones * 0x76u)) | x) & (ones * 0x80u)) == 0; + }; + if (n >= 8) + { + for (std::size_t i = 0; i + 8 < n; i += 8) + { + if (!digits(p + i)) + { + return false; + } + } + return digits(p + n - 8); // (overlapping the bytes before: they are digits) + } + for (std::size_t i = 0; i < n; ++i) + { + if (!is_digit(p[i])) + { + return false; + } + } + return true; +} + +/// Whether a float token has the shape that the parser records for most of +/// them: [-]digits.digits, with the numbers of digits of the node's layout +/// (extra). Then the token is a number of that layout, and the general scan +/// that follows is not needed. +inline bool plain_float_token(const unsigned char* s, std::size_t len, std::uint16_t extra) noexcept +{ + const std::size_t int_digits = extra & 0xFFu; + const std::size_t frac_digits = extra >> 8u; + if (int_digits == 0 || int_digits >= 255 || frac_digits == 0 || frac_digits >= 255 || len == 0) + { + return false; + } + const std::size_t negative = *s == '-' ? 1 : 0; + if (len != negative + int_digits + 1 + frac_digits) + { + return false; + } + const unsigned char* const p = s + negative; + return !(int_digits > 1 && *p == '0') && p[int_digits] == '.' && all_digits(p, int_digits) && all_digits(p + int_digits + 1, frac_digits); +} + /// whether [s, s + len) is a JSON number (the grammar the parser accepts) inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) { @@ -276,11 +351,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to } else if (*p >= '1' && *p <= '9') { - do - { - ++p; - } - while (p != e && is_digit(*p)); + p = digit_run(p + 1, e); } else { @@ -292,10 +363,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to if (p != e && *p == '.') { const unsigned char* const f0 = ++p; - while (p != e && is_digit(*p)) - { - ++p; - } + p = digit_run(p, e); if (p == f0) { return false; @@ -406,10 +474,15 @@ struct float_range template bool check_distinct_ranges(std::vector& ranges, Check check) { - std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + // (save() writes the ranges in the order of the nodes, which is sorted already) + const auto before = [](const byte_range & a, const byte_range & b) { return a.off != b.off ? a.off < b.off : a.len < b.len; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; for (std::size_t i = 0; i < ranges.size(); ++i) { @@ -431,7 +504,8 @@ bool check_distinct_ranges(std::vector& ranges, Check check) /// record layouts that match it), tokens must not overlap. inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) { - std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + // (save() writes the tokens in the order of the nodes, which is sorted already) + const auto before = [](const float_range & a, const float_range & b) { if (a.off != b.off) { @@ -442,7 +516,11 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned return a.len < b.len; } return a.extra < b.extra; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; std::size_t i = 0; while (i < ranges.size()) @@ -452,14 +530,18 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned { return false; } - number_token t{}; const unsigned char* const s = text + r.off; - if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + std::uint16_t layout = r.extra; + if (!plain_float_token(s, r.len, r.extra)) { - return false; + number_token t{}; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + layout = float_layout(t); } - // the digit layout the parser records (or "many", as compaction writes it) - const std::uint16_t layout = float_layout(t); for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) { if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index dbfbdc7a9..9dede9b09 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -29,6 +29,13 @@ // NLOHMANN_VIEW_NO_TARGET_ATTRIBUTE (set by the GCC module interface, where GCC // ignores the target attribute and then rejects the SSSE3 intrinsics) keeps the // check portable without disabling the SSE2 code. +// +// The vector UTF-8 check (utf8_lookup4) is adapted from simdjson +// (https://github.com/simdjson/simdjson, Daniel Lemire, Geoff Langdale, John +// Keiser, and contributors, available under the Apache 2.0 and MIT licenses; +// used here under the MIT license): the "lookup4" algorithm and tables of +// J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per +// Byte" (2021). #if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && NLOHMANN_VIEW_LITTLE_ENDIAN #include #define NLOHMANN_VIEW_NEON 1 diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index d413a55b9..0c6c7410b 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -763,8 +763,12 @@ class basic_json_view template friend class basic_json_document; template friend class basic_json_view; template friend class detail::view::editor; + template friend class detail::view::view_side; friend iterator; + /// whether the view is one of an editable document + static constexpr bool is_editable = Editable; + basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 98d6045fa..10b42dac1 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10970,6 +10970,13 @@ NLOHMANN_JSON_NAMESPACE_END // use std::from_chars or std::strtold. They are free functions so the lexer // stays focused on scanning (see lexer::convert_number()) and so that other // parsers of JSON text can convert tokens exactly like it does. +// +// The Eisel-Lemire conversion (eisel_lemire()), the digit comparison for the +// cases it cannot decide, and the eight-digits-at-a-time helpers +// (is_eight_digits(), parse_eight_digits()) are adapted from fast_float +// (https://github.com/fastfloat/fast_float, Daniel Lemire and contributors, +// available under the MIT, Apache 2.0, and Boost Software licenses; used here +// under the MIT license). The table of powers of five is in pow5_table.hpp. NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -12078,6 +12085,13 @@ NLOHMANN_JSON_NAMESPACE_END // NLOHMANN_JSON_NO_TARGET_ATTRIBUTE (for compilers that ignore the target // attribute and then reject the SSSE3 intrinsics) keeps the check portable // without disabling the SSE2 code. +// +// The vector UTF-8 check (utf8_lookup4) is adapted from simdjson +// (https://github.com/simdjson/simdjson, Daniel Lemire, Geoff Langdale, John +// Keiser, and contributors, available under the Apache 2.0 and MIT licenses; +// used here under the MIT license): the "lookup4" algorithm and tables of +// J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per +// Byte" (2021). #if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #include #define JSON_SCAN_NEON 1 @@ -32495,6 +32509,11 @@ NLOHMANN_JSON_NAMESPACE_END #include // conditional +// The shortest-digit conversion of doubles (see zmij.hpp) and the writing of +// its digits, including the conversion of the digits to text by Xiang JunBo +// and the SIMD instruction sequence by Dougall Johnson, are ported from Zmij +// by Victor Zverovich (https://github.com/vitaut/zmij, MIT license). +// // SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 // digits of a double at once (unless JSON_NO_SIMD selects the portable code) #if !defined(JSON_NO_SIMD) && (defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC))) diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 90593b2d1..c4636da14 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -688,6 +688,13 @@ NLOHMANN_JSON_NAMESPACE_END // NLOHMANN_VIEW_NO_TARGET_ATTRIBUTE (set by the GCC module interface, where GCC // ignores the target attribute and then rejects the SSSE3 intrinsics) keeps the // check portable without disabling the SSE2 code. +// +// The vector UTF-8 check (utf8_lookup4) is adapted from simdjson +// (https://github.com/simdjson/simdjson, Daniel Lemire, Geoff Langdale, John +// Keiser, and contributors, available under the Apache 2.0 and MIT licenses; +// used here under the MIT license): the "lookup4" algorithm and tables of +// J. Keiser and D. Lemire, "Validating UTF-8 In Less Than One Instruction Per +// Byte" (2021). #if !defined(JSON_NO_SIMD) && defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) && NLOHMANN_VIEW_LITTLE_ENDIAN #include #define NLOHMANN_VIEW_NEON 1 @@ -2472,15 +2479,367 @@ NLOHMANN_JSON_NAMESPACE_END -#include // sort, stable_sort +#include // is_sorted, sort, stable_sort +#include // array #include // size_t +#include // int64_t, uint32_t, uint64_t +#include // memcpy +#include // equal_to #include // string -#include // move, pair +#include // move, pair, swap #include // vector // #include +// #include + // #include +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits +#include // string +#include // integral_constant + +// #include +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/*! +@brief locate the decimal point and the end of the mantissa of a float token + +Also checks that the token is a JSON number. Tokens of the parser and of edits +always are; an image loaded with image_check::bounds can hold any bytes, which +must not reach the conversion (it expects a well-formed token). +*/ +inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept +{ + const auto digit = [last](const char* q) + { + return q != last && is_digit(static_cast(*q)); + }; + const char* p = first; + p += (p != last && *p == '-') ? 1 : 0; + if (!digit(p) || (*p == '0' && digit(p + 1))) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + dot = std::string::npos; + if (p != last && *p == '.') + { + dot = static_cast(p - first); + if (!digit(++p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + mantissa_end = static_cast(p - first); + if (p != last && (*p == 'e' || *p == 'E')) + { + ++p; + p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; + if (!digit(p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + return p == last; +} + +/*! +@brief the value of the float token of a node, as parse() converts it + +Uses the lexer's conversion (detail::convert_float), so that the values are +bit-identical to parse(): float and double are converted without allocation +and independent of the locale. A token that is not a JSON number (only in a +damaged image loaded with image_check::bounds) yields 0. +*/ +template +NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) +{ + const char* const last = first + n.len; + std::size_t dot = 0; + std::size_t mantissa_end = 0; + if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) + { + return FloatType{}; + } + return convert_float(first, last, dot, mantissa_end); +} + +/*! +@brief the digits of a float token with at most 19 digits, from its layout + +The digit layout recorded while parsing says where the integer digits, the +fraction digits, and the exponent are, so the digits are read eight at a +time without scanning. + +@param[in] p first character of the token +@param[in] e end of the token +@param[in] limit end of the readable memory (the source text) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + const bool negative = *p == '-'; + p += negative ? 1 : 0; + std::uint64_t w = parse_upto19(p, int_digits, limit); + p += int_digits; + std::int64_t q = 0; + if (frac_digits != 0) + { + w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); + p += 1 + frac_digits; + q = -static_cast(frac_digits); + } + if (p != e) + { + // [eE][+-]digits; huge exponents saturate (the parser rejected + // overflow). The token is not read beyond e, and the digits are taken + // as unsigned, so that a token that is not well-formed (a damaged + // image loaded with image_check::bounds) yields a wrong value, but no + // overflow. + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; + std::int64_t exp_value = 0; + for (; p != e; ++p) + { + if (exp_value < 0x10000000) + { + exp_value = (exp_value * 10) + static_cast(*p - '0'); + } + } + q += exp_negative ? -exp_value : exp_value; + } + + float_significand d; + d.w = w; + d.exponent = q; + d.negative = negative; + return d; +} + +/*! +@brief the value of a float token with at most 19 digits, from its layout + +The result is correctly rounded by the lexer's conversion +(detail::decimal_to_float(): Clinger's fast path where both operands are +exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 +digits), so it is the value parse() produces. +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); +} + +/// the value of a float set by an edit: its token (the shortest round-trip +/// text, or "nan", "inf", "-inf") in the edit arena +template +NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) +{ + if (token[0] == 'n') + { + return std::numeric_limits::quiet_NaN(); + } + if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) + { + return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); + } + return float_value(token, n); +} + +/// the value of the float token of a node, as parse() converts it; floats and +/// doubles with at most 19 digits are converted from the digit layout +template +FloatType float_value(const document_data& d, const node& n) +{ + if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) + { + return edited_float(d.str(n), n); + } + return float_value(d, n, std::integral_constant::value> {}); +} + +template +FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") + { + // (a float token not written by an edit is in the text) + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + } + return float_value(d.str(n), n); +} + +template +FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) +{ + return float_value(d.str(n), n); +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // min +#include // size_t +#include // memcmp, strlen +#include // basic_string + +// #include +// #include + + +#if NLOHMANN_VIEW_HAS_CPP_17 + #include // string_view +#endif +#ifndef JSON_NO_IO + #include // ostream +#endif + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +#if NLOHMANN_VIEW_HAS_CPP_17 +using string_ref = std::string_view; +#else +/// minimal C++11 stand-in for std::string_view +class string_ref +{ + public: + using size_type = std::size_t; + using const_iterator = const char*; + + string_ref() noexcept = default; + // s must be null-terminated, as for std::string_view(const char*) + // flawfinder: ignore + string_ref(const char* s) : m_data(s), m_size(std::strlen(s)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + string_ref(const char* s, std::size_t n) noexcept : m_data(s), m_size(n) {} + template + string_ref(const std::basic_string& s) noexcept : m_data(s.data()), m_size(s.size()) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + + const char* data() const noexcept + { + return m_data; + } + std::size_t size() const noexcept + { + return m_size; + } + std::size_t length() const noexcept + { + return m_size; + } + bool empty() const noexcept + { + return m_size == 0; + } + const char* begin() const noexcept + { + return m_data; + } + const char* end() const noexcept + { + return m_data + m_size; + } + char operator[](std::size_t i) const noexcept + { + return m_data[i]; + } + + template + explicit operator std::basic_string() const + { + return std::basic_string(m_data, m_size); + } + + friend bool operator==(string_ref a, string_ref b) noexcept + { + return a.m_size == b.m_size && (a.m_size == 0 || std::memcmp(a.m_data, b.m_data, a.m_size) == 0); + } + friend bool operator!=(string_ref a, string_ref b) noexcept + { + return !(a == b); + } + friend bool operator<(string_ref a, string_ref b) noexcept + { + const int c = std::memcmp(a.m_data, b.m_data, (std::min)(a.m_size, b.m_size)); + return c != 0 ? c < 0 : a.m_size < b.m_size; + } +#ifndef JSON_NO_IO + friend std::ostream& operator<<(std::ostream& o, string_ref s) + { + return o.write(s.m_data, static_cast(s.m_size)); + } +#endif + + private: + const char* m_data = ""; + std::size_t m_size = 0; +}; +#endif + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -2495,17 +2854,355 @@ namespace view // value, at the position of the first occurrence), and in document order if // the object type keeps an order (ordered_json), by key otherwise. +/// whether n bytes are equal; the short texts of JSON (keys, numbers, most +/// strings) are compared by two overlapping loads instead of a library call +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_equal(const char* x, const char* y, std::size_t n) noexcept +{ + const auto load8 = [](const char* p) + { + std::uint64_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + const auto load4 = [](const char* p) + { + std::uint32_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + if (n >= 8) + { + for (std::size_t i = 0; i + 8 < n; i += 8) + { + if (load8(x + i) != load8(y + i)) + { + return false; + } + } + return load8(x + n - 8) == load8(y + n - 8); + } + if (n >= 4) + { + return load4(x) == load4(y) && load4(x + n - 4) == load4(y + n - 4); + } + for (std::size_t i = 0; i < n; ++i) + { + if (x[i] != y[i]) + { + return false; + } + } + return true; +} + +/// whether the n1 bytes at x are before the n2 bytes at y (memcmp() order, then the shorter first); +/// the first difference is found eight bytes at a time +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_less(const char* x, std::size_t n1, const char* y, std::size_t n2) noexcept +{ + const std::size_t n = n1 < n2 ? n1 : n2; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t u = read_eight_bytes(x + i); + const std::uint64_t v = read_eight_bytes(y + i); + if (u != v) + { + const auto shift = static_cast(count_trailing_zeros(u ^ v)) & ~7u; // (the first byte that differs) + return ((u >> shift) & 0xFFu) < ((v >> shift) & 0xFFu); + } + } + for (; i < n; ++i) + { + if (x[i] != y[i]) + { + return static_cast(x[i]) < static_cast(y[i]); + } + } + return n1 < n2; +} + +/// whether the key a is before the key b +template +NLOHMANN_VIEW_ALWAYS_INLINE bool key_less(const Key& a, const Key& b) noexcept +{ + return bytes_less(a.data(), a.size(), b.data(), b.size()); +} + +/// sorts the indices order[0..n) of the members m by key; equal keys stay in +/// document order (small objects with an insertion sort, which needs no memory) +template +void sort_by_key(std::uint32_t* order, std::size_t n, const Entry* m) +{ + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + const std::uint32_t x = order[i]; + std::size_t j = i; + for (; j > 0 && key_less(m[x].first, m[order[j - 1]].first); --j) + { + order[j] = order[j - 1]; + } + order[j] = x; + } + } + else + { + std::stable_sort(order, order + n, [m](std::uint32_t x, std::uint32_t y) + { + return key_less(m[x].first, m[y].first); + }); + } +} + +/// The members of an object as parse() keeps them: one per key, the last +/// value at the position of the first occurrence, in the order of the +/// comparison (sorted by key, or, if the object type keeps the order, the order +/// of first occurrence). The members m[0..n) are in document order; this +/// appends to out the indices of the member of each key that has its value +/// (its last one), in the order of the comparison. Most objects have unique +/// keys. scratch is working memory, kept by the caller so that its capacity +/// is reused. +template +void resolve_indices(const Entry* m, std::size_t n, bool ordered, std::vector& out, std::vector& scratch) +{ + const std::size_t start = out.size(); + out.resize(start + n); + std::uint32_t* const order = out.data() + start; + sort_by_key(order, n, m); + if (!ordered) + { + // the last of each run of equal keys + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + order[w++] = order[last]; + i = last + 1; + } + out.resize(start + w); + return; + } + // (document order) the value of the last occurrence at the first occurrence of each key + constexpr std::uint32_t none = 0xFFFFFFFFu; + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + if (last != i) + { + if (!any) + { + scratch.assign(n, none); + for (std::size_t k = 0; k < i; ++k) + { + scratch[order[k]] = order[k]; + } + any = true; + } + scratch[order[i]] = order[last]; // (stable: order[i] is the first occurrence) + } + else if (any) + { + scratch[order[i]] = order[i]; + } + i = last + 1; + } + if (!any) + { + // unique keys: document order + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + return; + } + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (scratch[i] != none) + { + order[w++] = scratch[i]; + } + } + out.resize(start + w); +} + +/// whether the keys of n members are the same, in the same order +template +bool same_keys(const EntryA* a, const EntryB* b, std::size_t n) +{ + for (std::size_t i = 0; i < n; ++i) + { + if (!(a[i].first == b[i].first)) + { + return false; + } + } + return true; +} + +/// The members of an object as parse() keeps them (one per key: the last +/// value at the position of the first occurrence), sorted by key or, if the +/// object type keeps the order, in the order of first occurrence. The +/// members are given in document order, at the end of m (from the position +/// base on); most objects have unique keys, in which case they are only sorted +/// (for small objects with an insertion sort, which keeps equal keys in order +/// and needs no memory). scratch is working memory, kept by the caller. +template +void resolve_members(std::vector>& m, std::size_t base, bool ordered, std::vector& scratch) +{ + using entry = std::pair; + const std::size_t n = m.size() - base; + entry* const first = m.data() + base; + if (ordered) + { + // (document order) later values of a key go to its first occurrence; + // scratch holds the order by key, then whether a member is dropped + constexpr std::uint32_t dropped = 1; + scratch.assign(2 * n, 0); + std::uint32_t* const order = scratch.data(); + std::uint32_t* const drop = order + n; + sort_by_key(order, n, first); + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[order[last + 1]].first == first[order[i]].first) + { + ++last; + } + if (last != i) + { + first[order[i]].second = first[order[last]].second; + for (std::size_t k = i + 1; k <= last; ++k) + { + drop[order[k]] = dropped; + } + any = true; + } + i = last + 1; + } + if (any) + { + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (drop[i] != dropped) + { + if (w != i) + { + first[w] = std::move(first[i]); + } + ++w; + } + } + m.erase(m.begin() + static_cast(base + w), m.end()); + } + return; + } + // sorted by key; equal keys stay in document order + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + entry x = std::move(first[i]); + std::size_t j = i; + for (; j > 0 && key_less(x.first, first[j - 1].first); --j) + { + first[j] = std::move(first[j - 1]); + } + first[j] = std::move(x); + } + } + else + { + std::stable_sort(first, first + n, [](const entry & x, const entry & y) + { + return key_less(x.first, y.first); + }); + } + // the first key of a run with the last value + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[last + 1].first == first[i].first) + { + ++last; + } + if (w != i || last != i) + { + first[w] = entry(first[i].first, std::move(first[last].second)); + } + ++w; + i = last + 1; + } + m.erase(m.begin() + static_cast(base + w), m.end()); +} + +/// room for k more entries, growing geometrically (reserve(size() + k) alone would +/// reallocate for each object nested in another) +template +void reserve_more(std::vector& v, std::size_t k) +{ + const std::size_t needed = v.size() + k; + if (needed > v.capacity()) + { + v.reserve((std::max)(needed, 2 * v.capacity())); + } +} + /// one side of a comparison: a view template class view_side { public: using string_view_t = typename View::string_view_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using nav = navigation; + + /// the position in the elements of an array + struct cursor + { + const node* pos = nullptr; + }; + + view_side() noexcept + : m_view() + {} explicit view_side(const View& v) noexcept : m_view(v) {} + /// the document and the node of the view (nullptr: a discarded view) + const document_data& document() const noexcept + { + return *m_view.m_doc; + } + + const node* value_node() const noexcept + { + return m_view.m_node; + } + + static constexpr bool is_editable = View::is_editable; + value_t type() const noexcept { return m_view.type(); @@ -2521,6 +3218,32 @@ class view_side return m_view.get_string(); } + bool boolean() const noexcept + { + return (m_view.m_node->flags & node_flags::is_true) != 0; + } + + number_integer_t integer() const noexcept + { + return static_cast(static_cast(integer_bits(*m_view.m_node))); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return static_cast(integer_bits(*m_view.m_node)); + } + + number_float_t floating() const + { + return float_value(*m_view.m_doc, *m_view.m_node); + } + + /// the token of a float + string_view_t float_token() const noexcept + { + return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); + } + /// a number, boolean, or null as a basic_json value (no allocation) BasicJsonType scalar() const { @@ -2545,60 +3268,47 @@ class view_side } } - void elements(std::vector& out) const + /// start at the first element of this array + void enter(cursor& c) const noexcept { - out.reserve(m_view.size()); - for (const View e : m_view) + c.pos = nav::first(*m_view.m_doc, m_view.m_node); + } + + /// the element at c (and advance) + view_side take(cursor& c) const noexcept + { + const node* const v = nav::value(c.pos); + c.pos = document_data::after(c.pos); + return view_side(View(m_view.m_doc, v)); + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const noexcept + { + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = 0; i < n; ++i) { - out.emplace_back(e); + out[i] = string_view_t(m_view.m_doc->str(*pos), pos->len); + pos = document_data::after(pos + 1); } } - /// the members as parse() keeps them: one per key, the last value at the - /// position of the first occurrence; in that order, or sorted by key - void members(std::vector>& out, bool ordered) const + /// append the members to out, in document order; whether their keys ascend + /// (then they are unique, and in the order of the comparison of a std::map) + bool members(std::vector>& out, bool /*ordered*/) const { - struct member + const std::size_t base = out.size(); + reserve_more(out, m_view.m_node->len); + bool sorted = true; + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = m_view.m_node->len; i != 0; --i) { - string_view_t key; - View value; - std::size_t position; - }; - std::vector all; - all.reserve(m_view.size()); - std::size_t position = 0; - for (auto it = m_view.begin(); it != m_view.end(); ++it) - { - all.push_back(member{it.key(), it.value(), position++}); - } - std::stable_sort(all.begin(), all.end(), [](const member & a, const member & b) - { - return a.key < b.key; - }); - std::vector unique; - unique.reserve(all.size()); - for (std::size_t i = 0; i < all.size();) - { - std::size_t last = i; - while (last + 1 < all.size() && all[last + 1].key == all[i].key) - { - ++last; - } - unique.push_back(member{all[i].key, all[last].value, all[i].position}); - i = last + 1; - } - if (ordered) - { - std::sort(unique.begin(), unique.end(), [](const member & a, const member & b) - { - return a.position < b.position; - }); - } - out.reserve(unique.size()); - for (const member& m : unique) - { - out.emplace_back(m.key, view_side(m.value)); + const string_view_t key(m_view.m_doc->str(*pos), pos->len); + sorted = sorted && (out.size() == base || key_less(out.back().first, key)); + out.emplace_back(key, view_side(View(m_view.m_doc, nav::value(pos + 1)))); + pos = document_data::after(pos + 1); } + return sorted; } private: @@ -2611,6 +3321,17 @@ class json_side { public: using string_view_t = StringView; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + /// the position in the elements of an array + struct cursor + { + typename BasicJsonType::const_iterator it{}; + }; + + json_side() noexcept = default; explicit json_side(const BasicJsonType& j) noexcept : m_json(&j) @@ -2632,45 +3353,428 @@ class json_side return string_view_t(s.data(), s.size()); } + bool boolean() const noexcept + { + return *m_json->template get_ptr(); + } + + number_integer_t integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_float_t floating() const noexcept + { + return *m_json->template get_ptr(); + } + BasicJsonType scalar() const { return *m_json; } - void elements(std::vector& out) const + void enter(cursor& c) const { - out.reserve(m_json->size()); - for (const auto& e : *m_json) + c.it = m_json->cbegin(); + } + + json_side take(cursor& c) const + { + const json_side r(*c.it); + ++c.it; + return r; + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const + { + auto it = m_json->cbegin(); + for (std::size_t i = 0; i < n; ++i, ++it) { - out.emplace_back(e); + out[i] = string_view_t(it.key().data(), it.key().size()); } } - void members(std::vector>& out, bool ordered) const + /// append the members to out, in the order of the comparison (the keys of a + /// basic_json are unique); true + bool members(std::vector>& out, bool ordered) const { - out.reserve(m_json->size()); + const std::size_t base = out.size(); + reserve_more(out, m_json->size()); for (auto it = m_json->cbegin(); it != m_json->cend(); ++it) { out.emplace_back(string_view_t(it.key().data(), it.key().size()), json_side(it.value())); } if (!ordered) { - std::sort(out.begin(), out.end(), [](const std::pair& a, const std::pair& b) + const auto less = [](const std::pair& a, const std::pair& b) { - return a.first < b.first; - }); + return key_less(a.first, b.first); + }; + const auto first = out.begin() + static_cast(base); + if (!std::is_sorted(first, out.end(), less)) + { + std::sort(first, out.end(), less); + } + } + return true; + } + + private: + const BasicJsonType* m_json = nullptr; +}; + +/*! +@brief the values at two nodes, found equal by walking both documents in the +same order + +Two values are equal if they have the same type, the same number token or +string, the same elements, and the same members with the same keys in the same +order: a value that matches this way is equal under the rules above whatever +the object type and duplicate keys (the last value of a key then wins on +both sides, and their order of first occurrence is the same). The walk does +not find every equal pair (numbers spelled differently, members in another +order, duplicate keys with other values), only the ones that are the same +text; it stops at the first difference and then says nothing. Containers +nested deeper than the budget are not walked. + +The walk goes through the document in its order, the comparison that follows +goes through the members of an object by key. So that the walk never reaches +a difference later than the comparison would, and never costs more than it, +it enters an object only if the two orders agree (the keys ascend, or the +object type keeps the order) or if the walk is cheaper than preparing the +comparison of the object (all values are scalars). +*/ +/// the containers that lockstep found different, from the outermost to the +/// innermost: the comparison that goes on from there does not walk them again +struct lockstep_path +{ + /// the nesting depth up to which lockstep walks (the recursion is bounded) + static constexpr unsigned budget() noexcept + { + return 48; + } + + /// a pair of containers that differ; for arrays, the elements where the walk stopped + /// (the ones before are equal) + struct entry + { + const node* a; + const node* b; + const node* element_a; + const node* element_b; + std::uint32_t index; + }; + + /// the number of entries that a path can hold (a container deeper than the + /// budget of the walk is not entered) + static constexpr std::size_t levels() noexcept + { + return 50; + } + + explicit lockstep_path(entry* storage) noexcept + : nodes(storage) + {} + + entry* nodes; ///< levels() entries, written before they are read + unsigned length = 0; ///< pairs recorded by the last walk + unsigned next = 0; ///< the pair that the comparison enters next + const entry* resume = nullptr; ///< the entry of the pair that was just looked at, if it is in the path +}; + +template +class lockstep +{ + public: + lockstep(const document_data& a, const document_data& b, lockstep_path& path, bool ordered) noexcept + : m_a(a), m_b(b), m_path(path), m_ordered(ordered) + {} + + /// whether the values at a and b are the same (the nodes of the values, + /// not entries of a sequence) + bool same(const node* a, const node* b, unsigned budget) noexcept + { + if (a->kind != b->kind) + { + return false; + } + switch (static_cast(a->kind)) + { + case value_t::null: + return true; + case value_t::boolean: + return ((a->flags ^ b->flags) & node_flags::is_true) == 0; + case value_t::string: + return a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_float: // (the token, and the digit layout that the value is read with: they only differ in a damaged image) + return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_integer: + case value_t::number_unsigned: + return integer_bits(*a) == integer_bits(*b); + case value_t::array: + if (budget == 0 || a->len != b->len) + { + return false; + } + { + lockstep_path::entry stop{a, b, nullptr, nullptr, 0}; + if (elements(a, b, budget - 1, stop)) + { + return true; + } + record(stop, budget); + return false; + } + case value_t::object: + if (budget == 0 || a->len != b->len) + { + return false; + } + if (!visits_in_order(a) || !members(a, b, budget - 1)) + { + record(lockstep_path::entry{a, b, nullptr, nullptr, 0}, budget); + return false; + } + return true; + case value_t::binary: // LCOV_EXCL_LINE (not in a document) + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE } } private: - const BasicJsonType* m_json; + using nav_a = navigation; + using nav_b = navigation; + + /// note the containers a and b (at the nesting level that budget stands + /// for) as different + NLOHMANN_VIEW_NOINLINE void record(const lockstep_path::entry& stop, unsigned budget) const noexcept + { + const unsigned level = lockstep_path::budget() - budget; + m_path.nodes[level] = stop; + if (m_path.length == 0) + { + m_path.length = level + 1; // (the innermost container is the first to be left) + } + } + + /// whether the object @a a of a document that cannot be edited has scalar values only + static bool scalar_values_only(const node* a, std::false_type /*editable*/) noexcept + { + return a->next == 1 + (2 * static_cast(a->len)); + } + + /// (the nodes of an editable document can be elsewhere: not decided) + static bool scalar_values_only(const node* /*a*/, std::true_type /*editable*/) noexcept + { + return false; + } + + /// Whether the comparison that follows visits the members of the object a in the + /// order of the document: if the object type keeps the order, if the keys + /// ascend (it sorts by key), or if all values are scalars (then the walk costs + /// less than collecting and sorting the members, whichever member differs). + /// Otherwise a difference could be found by the walk later than by the + /// comparison, which is not worth the walk. + bool visits_in_order(const node* a) const noexcept + { + if (m_ordered || a->len < 2) + { + return true; + } + if (scalar_values_only(a, std::integral_constant {})) + { + return true; + } + const node* k = nav_a::first(m_a, a); + string_ref previous(m_a.str(*k), k->len); + for (std::uint32_t i = a->len - 1; i != 0; --i) + { + k = document_data::after(k + 1); + const string_ref key(m_a.str(*k), k->len); + if (!key_less(previous, key)) + { + return false; + } + previous = key; + } + return true; + } + + bool elements(const node* a, const node* b, unsigned budget, lockstep_path::entry& stop) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (!same(nav_a::value(pa), nav_b::value(pb), budget)) + { + stop.element_a = pa; + stop.element_b = pb; + stop.index = a->len - i; + return false; + } + pa = document_data::after(pa); + pb = document_data::after(pb); + } + return true; + } + + bool members(const node* a, const node* b, unsigned budget) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (pa->len != pb->len || !bytes_equal(m_a.str(*pa), m_b.str(*pb), pa->len)) + { + return false; + } + if (!same(nav_a::value(pa + 1), nav_b::value(pb + 1), budget)) + { + return false; + } + pa = document_data::after(pa + 1); + pb = document_data::after(pb + 1); + } + return true; + } + + const document_data& m_a; + const document_data& m_b; + lockstep_path& m_path; + bool m_ordered; ///< the object type keeps the order of the members }; +/// whether two views certainly have equal values (see lockstep); if not, the +/// path says where they differ +template +bool certainly_equal(const view_side& a, const view_side& b, lockstep_path& path, bool ordered) noexcept +{ + const node* const na = a.value_node(); + const node* const nb = b.value_node(); + if (na == nullptr || nb == nullptr) + { + return false; + } + if (na == nb) + { + return true; // (the same node of the same document: documents have distinct nodes) + } + path.resume = nullptr; + if (path.next < path.length && path.nodes[path.next].a == na && path.nodes[path.next].b == nb) + { + path.resume = &path.nodes[path.next]; + ++path.next; // (the walk found these different already) + return false; + } + path.length = 0; + const bool equal = lockstep::is_editable, view_side::is_editable>(a.document(), b.document(), path, ordered).same(na, nb, lockstep_path::budget()); + path.next = path.length != 0 ? 1 : 0; // (this pair is the first in the path) + path.resume = path.length != 0 ? &path.nodes[0] : nullptr; + return equal; +} + +/// (a view and a basic_json value: no walk) +template +bool certainly_equal(const A& /*a*/, const B& /*b*/, lockstep_path& /*path*/, bool /*ordered*/) noexcept +{ + return false; +} + +/// (arrays of two views) the elements where the walk stopped are the next pair: the ones before are equal +template +void resume_array(const lockstep_path& path, const view_side& a, const view_side& b, + typename view_side::cursor& cursor_a, typename view_side::cursor& cursor_b, std::size_t& next) noexcept +{ + if (path.resume != nullptr && path.resume->element_a != nullptr && path.resume->a == a.value_node() && path.resume->b == b.value_node()) + { + cursor_a.pos = path.resume->element_a; + cursor_b.pos = path.resume->element_b; + next = path.resume->index; + } +} + +template +void resume_array(const lockstep_path& /*path*/, const A& /*a*/, const B& /*b*/, CursorA& /*cursor_a*/, CursorB& /*cursor_b*/, std::size_t& /*next*/) noexcept +{} + +/// whether two floats are equal (the tokens of two views: equal text and digit layout is an equal value; they +/// only differ in a damaged image) +template +bool floats_equal(const view_side& a, const view_side& b) +{ + return (a.value_node()->extra == b.value_node()->extra && a.float_token() == b.float_token()) || std::equal_to()(a.floating(), b.floating()); +} + +template +bool floats_equal(const A& a, const B& b) +{ + return std::equal_to()(a.floating(), b.floating()); +} + +/// Whether two objects of a type that keeps the order are different because of the keys of +/// their first members, found without the preparation of the comparison: the comparison goes +/// through the members in the order of the first occurrence of their keys, so if the first +/// members have the same distinct keys and then one pair of keys differs, where neither is +/// one of the keys before it, the objects are different. (The keys that are the same as an +/// earlier key are the duplicates that the comparison drops, so those are not decided here.) +template +bool keys_differ_early(const SideA& a, const SideB& b) +{ + using string_view_t = typename SideA::string_view_t; + constexpr std::size_t limit = 4; + const std::size_t n = (std::min)((std::min)(a.size(), b.size()), limit); + std::array ka{}; + std::array kb{}; + a.first_keys(ka.data(), n); + b.first_keys(kb.data(), n); + for (std::size_t i = 0; i < n; ++i) + { + if (ka[i] != kb[i]) + { + for (std::size_t k = 0; k < i; ++k) + { + if (ka[i] == ka[k] || kb[i] == ka[k]) + { + return false; // (a duplicate: not decided) + } + } + return true; + } + } + return false; +} + +/// keys_differ_early() for a type that keeps the order of the members +template +bool keys_differ_early(const SideA& a, const SideB& b, std::true_type /*ordered*/) +{ + return keys_differ_early(a, b); +} + +/// (a type that sorts the members by key: not decided early) +template +bool keys_differ_early(const SideA& /*a*/, const SideB& /*b*/, std::false_type /*ordered*/) noexcept +{ + return false; +} + inline bool is_number(value_t t) noexcept { return t == value_t::number_integer || t == value_t::number_unsigned || t == value_t::number_float; } +/// the index_a/index_b of a frame of equal() whose members are compared in document order +constexpr std::size_t document_order = static_cast(-1); + /// whether two sides are equal; iterative, so that the nesting depth is /// limited by memory only template @@ -2681,20 +3785,52 @@ bool equal(const A& a0, const B& b0) struct frame { - std::vector elements_a{}; - std::vector elements_b{}; - std::vector> members_a{}; - std::vector> members_b{}; + A array_a{}; + B array_b{}; + typename A::cursor cursor_a{}; + typename B::cursor cursor_b{}; + std::size_t base_a = 0; ///< the members of an object: from this position in members_a + std::size_t base_b = 0; + std::size_t index_a = document_order; ///< their order in the comparison: from this position in indices (document_order: as they are) + std::size_t index_b = document_order; + std::size_t index_mark = 0; ///< the size of indices before this frame bool object = false; std::size_t next = 0; + std::size_t count = 0; }; std::vector stack; + std::vector indices; // the order of the members of the open objects, where it is not their order in the document + std::vector scratch; // working memory of resolve_indices() and resolve_members() + // the members of the open objects (the ones of an object follow those of the objects around it) + std::vector> members_a; + std::vector> members_b; + std::array path_storage; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + lockstep_path path(path_storage.data()); + { + // (values that differ in type or in the size of an array need no walk) + const value_t first = a0.type(); + const value_t second = b0.type(); + if (first != second && !(is_number(first) && is_number(second)) && first != value_t::discarded && second != value_t::discarded) + { + return false; + } + if (first == value_t::array && second == value_t::array && a0.size() != b0.size()) + { + return false; + } + } A a = a0; B b = b0; + // (pairs of views that are certainly equal need no further look) + if (certainly_equal(a, b, path, ordered)) + { + return true; + } for (;;) { const value_t ta = a.type(); const value_t tb = b.type(); + const bool numbers = is_number(ta) && is_number(tb); if (ta == value_t::discarded || tb == value_t::discarded) { // basic_json decides (JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON) @@ -2705,7 +3841,7 @@ bool equal(const A& a0, const B& b0) } else { - if (ta != tb && !(is_number(ta) && is_number(tb))) // numbers of different types can be equal + if (ta != tb && !numbers) // numbers of different types can be equal { return false; } @@ -2722,31 +3858,98 @@ bool equal(const A& a0, const B& b0) { return false; } - frame f; + if (ta == value_t::object && keys_differ_early(a, b, std::integral_constant::value> {})) + { + return false; + } + stack.emplace_back(); + frame& f = stack.back(); f.object = ta == value_t::object; if (f.object) { - a.members(f.members_a, ordered); - b.members(f.members_b, ordered); - if (f.members_a.size() != f.members_b.size()) + f.base_a = members_a.size(); + f.base_b = members_b.size(); + f.index_mark = indices.size(); + const bool sorted_a = a.members(members_a, ordered); + const bool sorted_b = b.members(members_b, ordered); + const std::size_t size_a = members_a.size() - f.base_a; + const std::size_t size_b = members_b.size() - f.base_b; + std::size_t count_a = size_a; + std::size_t count_b = size_b; + if (!sorted_a && !sorted_b && size_a == size_b && same_keys(members_a.data() + f.base_a, members_b.data() + f.base_b, size_a)) + { + // the same keys in the same order: one order of the comparison for both + f.index_a = indices.size(); + f.index_b = f.index_a; + resolve_indices(members_a.data() + f.base_a, size_a, ordered, indices, scratch); + count_a = indices.size() - f.index_a; + count_b = count_a; + } + else + { + if (!sorted_a) + { + resolve_members(members_a, f.base_a, ordered, scratch); + count_a = members_a.size() - f.base_a; + } + if (!sorted_b) + { + resolve_members(members_b, f.base_b, ordered, scratch); + count_b = members_b.size() - f.base_b; + } + } + f.count = count_a; + if (count_a != count_b) { return false; } } else { - a.elements(f.elements_a); - b.elements(f.elements_b); + f.array_a = a; + f.array_b = b; + a.enter(f.cursor_a); + b.enter(f.cursor_b); + f.count = a.size(); + resume_array(path, a, b, f.cursor_a, f.cursor_b, f.next); } - stack.push_back(std::move(f)); } - else if (a.scalar() != b.scalar()) // numbers (also of different types), null, boolean + else if (ta == value_t::boolean) + { + if (a.boolean() != b.boolean()) + { + return false; + } + } + else if (numbers && ta == tb) + { + // numbers of one type: as basic_json compares them + bool differ = false; + if (ta == value_t::number_integer) + { + differ = a.integer() != b.integer(); + } + else if (ta == value_t::number_unsigned) + { + differ = a.unsigned_integer() != b.unsigned_integer(); + } + else + { + differ = !floats_equal(a, b); + } + if (differ) + { + return false; + } + } + else if (numbers && !(a.scalar() == b.scalar())) // numbers of different types { return false; } + // (null equals null) } - // the next pair of values + // the next pair of values that is not certainly equal for (;;) { if (stack.empty()) @@ -2754,28 +3957,38 @@ bool equal(const A& a0, const B& b0) return true; } frame& f = stack.back(); - const std::size_t count = f.object ? f.members_a.size() : f.elements_a.size(); - if (f.next == count) + if (f.next == f.count) { + if (f.object) + { + members_a.erase(members_a.begin() + static_cast(f.base_a), members_a.end()); + members_b.erase(members_b.begin() + static_cast(f.base_b), members_b.end()); + indices.resize(f.index_mark); + } stack.pop_back(); // (f dangles now, and is bound again after continue) continue; } if (f.object) { - if (!(f.members_a[f.next].first == f.members_b[f.next].first)) + const auto& member_a = members_a[f.base_a + (f.index_a == document_order ? f.next : indices[f.index_a + f.next])]; + const auto& member_b = members_b[f.base_b + (f.index_b == document_order ? f.next : indices[f.index_b + f.next])]; + if (!(member_a.first == member_b.first)) { return false; } - a = f.members_a[f.next].second; - b = f.members_b[f.next].second; + a = member_a.second; + b = member_b.second; } else { - a = f.elements_a[f.next]; - b = f.elements_b[f.next]; + a = f.array_a.take(f.cursor_a); + b = f.array_b.take(f.cursor_b); } ++f.next; - break; + if (!certainly_equal(a, b, path, ordered)) + { + break; + } } } } @@ -3758,6 +4971,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -3831,10 +5046,9 @@ struct first_true : std::integral_constant < int, 1 + firs /// (type_error.316 with the same message), so that an editable document /// only holds valid UTF-8: the error is at the first byte that no /// well-formed sequence can continue with (Unicode, Table 3-7). -inline void check_utf8(const char* s, std::size_t n) +NLOHMANN_VIEW_ALWAYS_INLINE void check_utf8_bytewise(const unsigned char* p, std::size_t n, std::size_t i) { - const auto* const p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - for (std::size_t i = 0; i < n;) + while (i < n) { const unsigned char c = p[i]; if (c < 0x80) @@ -3864,6 +5078,43 @@ inline void check_utf8(const char* s, std::size_t n) } } +/// The long-string path of check_utf8(): skips well-formed runs with the +/// parser's string scan (which stops at a quote, a backslash, or a control +/// character, all legal here, and at ill-formed UTF-8); the byte-wise check +/// runs only from where the scan found ill-formed UTF-8, to report it. +NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std::size_t n) +{ + const unsigned char* const e = p + n; + const unsigned char* q = p; + for (;;) + { + q = scan_string_run(q, e); + if (q == e) + { + return; + } + if (*q >= 0x80) + { + check_utf8_bytewise(p, n, static_cast(q - p)); + return; // not reached: the scan stopped at ill-formed UTF-8 + } + ++q; // a quote, backslash, or control character + } +} + +inline void check_utf8(const char* s, std::size_t n) +{ + const auto* const p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + if (NLOHMANN_VIEW_UNLIKELY(n >= 32)) + { + check_utf8_scan(p, n); + return; + } + // short strings: the vector scan costs more than it saves; (the loop is + // inlined with a constant start, so this path is the plain byte-wise check) + check_utf8_bytewise(p, n, 0); +} + /*! @brief the edits of an editable basic_json_document @@ -4675,232 +5926,6 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // size_t -#include // int64_t, uint64_t -#include // numeric_limits -#include // string -#include // integral_constant - -// #include -// #include - -// #include - -// #include - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -/*! -@brief locate the decimal point and the end of the mantissa of a float token - -Also checks that the token is a JSON number. Tokens of the parser and of edits -always are; an image loaded with image_check::bounds can hold any bytes, which -must not reach the conversion (it expects a well-formed token). -*/ -inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept -{ - const auto digit = [last](const char* q) - { - return q != last && is_digit(static_cast(*q)); - }; - const char* p = first; - p += (p != last && *p == '-') ? 1 : 0; - if (!digit(p) || (*p == '0' && digit(p + 1))) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - dot = std::string::npos; - if (p != last && *p == '.') - { - dot = static_cast(p - first); - if (!digit(++p)) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - } - mantissa_end = static_cast(p - first); - if (p != last && (*p == 'e' || *p == 'E')) - { - ++p; - p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; - if (!digit(p)) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - } - return p == last; -} - -/*! -@brief the value of the float token of a node, as parse() converts it - -Uses the lexer's conversion (detail::convert_float), so that the values are -bit-identical to parse(): float and double are converted without allocation -and independent of the locale. A token that is not a JSON number (only in a -damaged image loaded with image_check::bounds) yields 0. -*/ -template -NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) -{ - const char* const last = first + n.len; - std::size_t dot = 0; - std::size_t mantissa_end = 0; - if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) - { - return FloatType{}; - } - return convert_float(first, last, dot, mantissa_end); -} - -/*! -@brief the digits of a float token with at most 19 digits, from its layout - -The digit layout recorded while parsing says where the integer digits, the -fraction digits, and the exponent are, so the digits are read eight at a -time without scanning. - -@param[in] p first character of the token -@param[in] e end of the token -@param[in] limit end of the readable memory (the source text) -*/ -NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - const bool negative = *p == '-'; - p += negative ? 1 : 0; - std::uint64_t w = parse_upto19(p, int_digits, limit); - p += int_digits; - std::int64_t q = 0; - if (frac_digits != 0) - { - w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); - p += 1 + frac_digits; - q = -static_cast(frac_digits); - } - if (p != e) - { - // [eE][+-]digits; huge exponents saturate (the parser rejected - // overflow). The token is not read beyond e, and the digits are taken - // as unsigned, so that a token that is not well-formed (a damaged - // image loaded with image_check::bounds) yields a wrong value, but no - // overflow. - ++p; - const bool exp_negative = p != e && *p == '-'; - p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; - std::int64_t exp_value = 0; - for (; p != e; ++p) - { - if (exp_value < 0x10000000) - { - exp_value = (exp_value * 10) + static_cast(*p - '0'); - } - } - q += exp_negative ? -exp_value : exp_value; - } - - float_significand d; - d.w = w; - d.exponent = q; - d.negative = negative; - return d; -} - -/*! -@brief the value of a float token with at most 19 digits, from its layout - -The result is correctly rounded by the lexer's conversion -(detail::decimal_to_float(): Clinger's fast path where both operands are -exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 -digits), so it is the value parse() produces. -*/ -template -NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); -} - -/// the value of a float set by an edit: its token (the shortest round-trip -/// text, or "nan", "inf", "-inf") in the edit arena -template -NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) -{ - if (token[0] == 'n') - { - return std::numeric_limits::quiet_NaN(); - } - if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) - { - return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); - } - return float_value(token, n); -} - -/// the value of the float token of a node, as parse() converts it; floats and -/// doubles with at most 19 digits are converted from the digit layout -template -FloatType float_value(const document_data& d, const node& n) -{ - if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) - { - return edited_float(d.str(n), n); - } - return float_value(d, n, std::integral_constant::value> {}); -} - -template -FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) -{ - const unsigned int_digits = n.extra & 0xFFu; - const unsigned frac_digits = n.extra >> 8u; - if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") - { - // (a float token not written by an edit is in the text) - const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - return float_value(d.str(n), n); -} - -template -FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) -{ - return float_value(d.str(n), n); -} - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include @@ -5139,6 +6164,81 @@ struct number_token bool is_float; ///< a fraction or an exponent }; +/// advance over ASCII digits, eight bytes at a time +inline const unsigned char* digit_run(const unsigned char* p, const unsigned char* e) noexcept +{ + constexpr std::uint64_t ones = 0x0101010101010101ull; + while (e - p >= 8) + { + // (the digits become 0..9; a byte above 9, or with bit 7 set, is no digit. A carry + // out of a byte with bit 7 set only affects the bytes above, after the first one + // that count_trailing_zeros finds.) + const std::uint64_t x = read_eight_bytes(p) ^ (ones * 0x30u); + const std::uint64_t not_digit = ((x + (ones * 0x76u)) | x) & (ones * 0x80u); + if (not_digit != 0) + { + return p + (static_cast(count_trailing_zeros(not_digit)) / 8); + } + p += 8; + } + while (p != e && is_digit(*p)) + { + ++p; + } + return p; +} + +/// whether the n bytes at p are ASCII digits (n > 0) +inline bool all_digits(const unsigned char* p, std::size_t n) noexcept +{ + const auto digits = [](const unsigned char* q) + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + const std::uint64_t x = read_eight_bytes(q) ^ (ones * 0x30u); + return (((x + (ones * 0x76u)) | x) & (ones * 0x80u)) == 0; + }; + if (n >= 8) + { + for (std::size_t i = 0; i + 8 < n; i += 8) + { + if (!digits(p + i)) + { + return false; + } + } + return digits(p + n - 8); // (overlapping the bytes before: they are digits) + } + for (std::size_t i = 0; i < n; ++i) + { + if (!is_digit(p[i])) + { + return false; + } + } + return true; +} + +/// Whether a float token has the shape that the parser records for most of +/// them: [-]digits.digits, with the numbers of digits of the node's layout +/// (extra). Then the token is a number of that layout, and the general scan +/// that follows is not needed. +inline bool plain_float_token(const unsigned char* s, std::size_t len, std::uint16_t extra) noexcept +{ + const std::size_t int_digits = extra & 0xFFu; + const std::size_t frac_digits = extra >> 8u; + if (int_digits == 0 || int_digits >= 255 || frac_digits == 0 || frac_digits >= 255 || len == 0) + { + return false; + } + const std::size_t negative = *s == '-' ? 1 : 0; + if (len != negative + int_digits + 1 + frac_digits) + { + return false; + } + const unsigned char* const p = s + negative; + return !(int_digits > 1 && *p == '0') && p[int_digits] == '.' && all_digits(p, int_digits) && all_digits(p + int_digits + 1, frac_digits); +} + /// whether [s, s + len) is a JSON number (the grammar the parser accepts) inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t) { @@ -5157,11 +6257,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to } else if (*p >= '1' && *p <= '9') { - do - { - ++p; - } - while (p != e && is_digit(*p)); + p = digit_run(p + 1, e); } else { @@ -5173,10 +6269,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to if (p != e && *p == '.') { const unsigned char* const f0 = ++p; - while (p != e && is_digit(*p)) - { - ++p; - } + p = digit_run(p, e); if (p == f0) { return false; @@ -5287,10 +6380,15 @@ struct float_range template bool check_distinct_ranges(std::vector& ranges, Check check) { - std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b) + // (save() writes the ranges in the order of the nodes, which is sorted already) + const auto before = [](const byte_range & a, const byte_range & b) { return a.off != b.off ? a.off < b.off : a.len < b.len; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; for (std::size_t i = 0; i < ranges.size(); ++i) { @@ -5312,7 +6410,8 @@ bool check_distinct_ranges(std::vector& ranges, Check check) /// record layouts that match it), tokens must not overlap. inline bool check_float_ranges(std::vector& ranges, const unsigned char* text) { - std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b) + // (save() writes the tokens in the order of the nodes, which is sorted already) + const auto before = [](const float_range & a, const float_range & b) { if (a.off != b.off) { @@ -5323,7 +6422,11 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned return a.len < b.len; } return a.extra < b.extra; - }); + }; + if (!std::is_sorted(ranges.begin(), ranges.end(), before)) + { + std::sort(ranges.begin(), ranges.end(), before); + } std::size_t end = 0; std::size_t i = 0; while (i < ranges.size()) @@ -5333,14 +6436,18 @@ inline bool check_float_ranges(std::vector& ranges, const unsigned { return false; } - number_token t{}; const unsigned char* const s = text + r.off; - if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + std::uint16_t layout = r.extra; + if (!plain_float_token(s, r.len, r.extra)) { - return false; + number_token t{}; + if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t)) + { + return false; + } + // the digit layout the parser records (or "many", as compaction writes it) + layout = float_layout(t); } - // the digit layout the parser records (or "many", as compaction writes it) - const std::uint16_t layout = float_layout(t); for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i) { if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu) @@ -7865,120 +8972,6 @@ class view_serializer NLOHMANN_JSON_NAMESPACE_END // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // min -#include // size_t -#include // memcmp, strlen -#include // basic_string - -// #include -// #include - - -#if NLOHMANN_VIEW_HAS_CPP_17 - #include // string_view -#endif -#ifndef JSON_NO_IO - #include // ostream -#endif - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -#if NLOHMANN_VIEW_HAS_CPP_17 -using string_ref = std::string_view; -#else -/// minimal C++11 stand-in for std::string_view -class string_ref -{ - public: - using size_type = std::size_t; - using const_iterator = const char*; - - string_ref() noexcept = default; - // s must be null-terminated, as for std::string_view(const char*) - // flawfinder: ignore - string_ref(const char* s) : m_data(s), m_size(std::strlen(s)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) - string_ref(const char* s, std::size_t n) noexcept : m_data(s), m_size(n) {} - template - string_ref(const std::basic_string& s) noexcept : m_data(s.data()), m_size(s.size()) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) - - const char* data() const noexcept - { - return m_data; - } - std::size_t size() const noexcept - { - return m_size; - } - std::size_t length() const noexcept - { - return m_size; - } - bool empty() const noexcept - { - return m_size == 0; - } - const char* begin() const noexcept - { - return m_data; - } - const char* end() const noexcept - { - return m_data + m_size; - } - char operator[](std::size_t i) const noexcept - { - return m_data[i]; - } - - template - explicit operator std::basic_string() const - { - return std::basic_string(m_data, m_size); - } - - friend bool operator==(string_ref a, string_ref b) noexcept - { - return a.m_size == b.m_size && (a.m_size == 0 || std::memcmp(a.m_data, b.m_data, a.m_size) == 0); - } - friend bool operator!=(string_ref a, string_ref b) noexcept - { - return !(a == b); - } - friend bool operator<(string_ref a, string_ref b) noexcept - { - const int c = std::memcmp(a.m_data, b.m_data, (std::min)(a.m_size, b.m_size)); - return c != 0 ? c < 0 : a.m_size < b.m_size; - } -#ifndef JSON_NO_IO - friend std::ostream& operator<<(std::ostream& o, string_ref s) - { - return o.write(s.m_data, static_cast(s.m_size)); - } -#endif - - private: - const char* m_data = ""; - std::size_t m_size = 0; -}; -#endif - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ @@ -8811,8 +9804,12 @@ class basic_json_view template friend class basic_json_document; template friend class basic_json_view; template friend class detail::view::editor; + template friend class detail::view::view_side; friend iterator; + /// whether the view is one of an editable document + static constexpr bool is_editable = Editable; + basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} diff --git a/tests/benchmarks/json_view/README.md b/tests/benchmarks/json_view/README.md index a0bd2afbc..8f56c2693 100644 --- a/tests/benchmarks/json_view/README.md +++ b/tests/benchmarks/json_view/README.md @@ -21,6 +21,10 @@ python3 tests/benchmarks/json_view/compare.py --data - The other libraries come from the system: pkg-config, or Homebrew (`brew install yyjson simdjson boost`). With `--download`, pinned releases are downloaded instead and checked against their SHA-256. Without Boost headers (or with `--no-boost`), the Boost.JSON columns are skipped, and the results say so. +- `--aligned` additionally builds and runs everything with all functions aligned to 64 bytes (`-mllvm + -align-all-functions=6` for Clang, `-falign-functions=64 -falign-loops=64` for GCC), and reports both result + tables as "default build" and "aligned build". Code layout alone can move a result by a few percent; a difference + that appears in only one of the two tables is a layout artifact. Off by default, which doubles the run time. - `--corpus file...` adds files to the corpus benchmark, e.g. those of [simdjson-data](https://github.com/simdjson/simdjson-data) or the [yyjson benchmark](https://github.com/ibireme/yyjson_benchmark). @@ -39,7 +43,7 @@ where `json_view` stands on another architecture; they are not meant for publica ## What is measured -`bench_view.cpp` runs four workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a +`bench_view.cpp` runs these workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a JSON-RPC request (`rpc`): | workload | what it does | @@ -48,6 +52,7 @@ JSON-RPC request (`rpc`): | traverse | parse, then visit every value, convert every number, touch every string and key | | select | parse, then read a few fields per record (e.g. id, user name, and retweet count of each tweet) | | dump | serialize a parsed document (compact) | +| parse+dump | build a fresh document, then serialize it (compact), in one timed call | `bench_corpus.cpp` runs parse, traverse, and dump on any list of files, so that no library is tuned to a handful of documents. Its dump also writes the numbers as they are in the input: `json_view` with `number_format::source`, and @@ -72,11 +77,32 @@ call follows an untimed call of the same engine: otherwise the engine after `jso cleaning up the tens of thousands of nodes `json::parse` just freed (with glibc, this made `json_view` look 1.7 times slower on citm_catalog traverse). +`json_view` is an index into the source text, not a tree of owning nodes. Its like-for-like peers are yyjson and +simdjson DOM (immutable, indexed, no per-node allocation). Boost.JSON and `json::parse` build mutable, owning DOMs and +are reference points for what that costs, not equals; simdjson On-Demand is lazy and forward-only, so it is the other +way round: it has no random access to pay for. All engines validate UTF-8 and syntax by default. + +Number conversion differs: `json_view` converts integers while parsing but converts floating-point numbers lazily, +when they are read or dumped; yyjson (default flags) and simdjson DOM convert every number while parsing. The cost of +float conversion therefore appears in the traverse and dump rows for `json_view`, and in the parse row for the others. +For float-heavy documents (canada), the fair reading is a sum of parse and use: the `traverse` workload already +includes the parse, and `parse+dump` measures a fresh document and its dump in one timed call; the `dump` row alone +leaves the conversion in `json_view` and out of the others. Eager conversion at parse time was prototyped as an opt-in +and measured: canada traverse (with its parse) was 4-7 % faster, but parse+dump 11-30 % slower and parse 36-90 % slower. It was +dropped, so lazy conversion is the only mode. + +`json_view` can also be edited: `json_editable_document` has the same read API, and the `(editable)` rows in +`bench_view` measure it on the same workloads. They are the fair counterpart of Boost.JSON and `json::parse`, which are +mutable too; the plain `json_view` rows are the counterpart of yyjson and simdjson DOM, which are not. `bench_edit` adds +the read-modify-write workloads. Read the tables with that in mind: an immutable-only library is not compared with an +editable `json_view` unless it says so. + The engines do not all offer the same features, which the numbers should be read with: | engine | document | random access | editable | notes | |---|---|---|---|---| | `json_view` | immutable index into the text | yes | no | a fresh document per parse; "reused" parses into the same document | +| `json_view` (editable) | index into the text plus edits | yes | yes, in place | `json_editable_document`; keeps member order and number spellings | | yyjson | immutable (`yyjson_read`) | yes | via a mutable copy | | | simdjson DOM | immutable, parser reused | yes | no | "fresh" uses a new parser per parse | | simdjson On-Demand | none: forward-only, lazy | no | no | only traverse and select | diff --git a/tests/benchmarks/json_view/bench_view.cpp b/tests/benchmarks/json_view/bench_view.cpp index 32d038649..71e19d68b 100644 --- a/tests/benchmarks/json_view/bench_view.cpp +++ b/tests/benchmarks/json_view/bench_view.cpp @@ -8,8 +8,9 @@ // Same-feature-set benchmark: read-only JSON documents with random access. // -// json_view nlohmann/json_view.hpp (fresh document per parse / reused) -// yyjson yyjson_read(): immutable document, random access +// json_view nlohmann/json_view.hpp (fresh document per parse / reused; +// "(editable)": json_editable_document, same read API) +// yyjson yyjson_read(): immutable document, random access // simdjson DOM dom::parser (reused, as recommended; "fresh": a new parser // per parse): immutable, random access // references (different feature sets): @@ -19,7 +20,11 @@ // // Workloads: parse (build + free), traverse (visit everything, convert every // number, touch every string and key), select (a few fields per document), -// dump (compact serialization of the parsed document). +// dump (compact serialization of the parsed document), and parse+dump (a fresh +// document, then its dump, in one timed call). json_view converts floating-point +// numbers lazily (when read or dumped), yyjson and simdjson DOM while parsing: +// for float-heavy documents, traverse (which includes the parse) and parse+dump +// are the fair reading of the parse and dump rows. // All engines run interleaved in every round, each timed call after an untimed // one of the same engine; the best round is reported. #include @@ -43,6 +48,8 @@ using nlohmann::json; using nlohmann::json_document; +using nlohmann::json_editable_document; +using nlohmann::json_editable_view; using nlohmann::json_view; static volatile double g_sink; @@ -55,7 +62,8 @@ struct stats // ---------------- traversal ---------------- -static void walk(json_view v, stats& st) +template +static void walk_view(V v, stats& st) { ++st.nodes; switch (v.type()) @@ -64,32 +72,42 @@ static void walk(json_view v, stats& st) for (auto it = v.begin(); it != v.end(); ++it) { st.str += it.key().size(); - walk(*it, st); + walk_view(*it, st); } break; case json::value_t::array: - for (const json_view e : v) + for (const V e : v) { - walk(e, st); + walk_view(e, st); } break; case json::value_t::string: st.str += v.get_string().size(); break; case json::value_t::number_integer: - st.num += static_cast(v.get()); + st.num += static_cast(v.template get()); break; case json::value_t::number_unsigned: - st.num += static_cast(v.get()); + st.num += static_cast(v.template get()); break; case json::value_t::number_float: - st.num += v.get(); + st.num += v.template get(); break; default: break; } } +static void walk(json_view v, stats& st) +{ + walk_view(v, st); +} + +static void walk(json_editable_view v, stats& st) +{ + walk_view(v, st); +} + static void walk(const json& j, stats& st) { ++st.nodes; @@ -300,20 +318,21 @@ static void walk(const boost::json::value& v, stats& st) // status (one tweet): id, user.screen_name, retweet_count // rpc: method, params.minuend, id -static double pick(const std::string& name, json_view r) +template +static double pick_view(const std::string& name, V r) { double acc = 0; if (name == "twitter" || name == "status") { - auto one = [&](json_view s) + auto one = [&](V s) { - acc += static_cast(s["id"].get()); + acc += static_cast(s["id"].template get()); acc += static_cast(s["user"]["screen_name"].get_string().size()); - acc += static_cast(s["retweet_count"].get()); + acc += static_cast(s["retweet_count"].template get()); }; if (name == "twitter") { - for (const json_view s : r["statuses"]) + for (const V s : r["statuses"]) { one(s); } @@ -325,20 +344,20 @@ static double pick(const std::string& name, json_view r) } else if (name == "citm_catalog") { - for (const json_view p : r["performances"]) + for (const V p : r["performances"]) { - acc += static_cast(p["id"].get() + p["eventId"].get() + p["seatCategories"].size()); + acc += static_cast(p["id"].template get() + p["eventId"].template get() + p["seatCategories"].size()); } acc += static_cast(r["events"].size()); } else if (name == "canada") { - const json_view g = r["features"][0]["geometry"]; + const V g = r["features"][0]["geometry"]; acc += static_cast(r["type"].get_string().size() + g["type"].get_string().size() + g["coordinates"].size()); } else if (name == "jeopardy") { - for (const json_view q : r) + for (const V q : r) { acc += q["round"].get_string() == "Final Jeopardy!" ? 1 : 0; acc += static_cast(q["category"].get_string().size()); @@ -347,11 +366,21 @@ static double pick(const std::string& name, json_view r) else if (name == "rpc") { acc += static_cast(r["method"].get_string().size()); - acc += static_cast(r["params"]["minuend"].get() + r["id"].get()); + acc += static_cast(r["params"]["minuend"].template get() + r["id"].template get()); } return acc; } +static double pick(const std::string& name, json_view r) +{ + return pick_view(name, r); +} + +static double pick(const std::string& name, json_editable_view r) +{ + return pick_view(name, r); +} + static double pick(const std::string& name, const json& r) { double acc = 0; @@ -634,15 +663,35 @@ int main(int argc, char** argv) walk(json::parse(dc.text), a); auto d = json_document::parse(dc.text); walk(d.root(), b); + auto ed = json_editable_document::parse(dc.text); + stats e; + walk(ed.root(), e); yyjson_doc* y = yyjson_read(dc.text.data(), dc.text.size(), 0); walk(yyjson_doc_get_root(y), c); simdjson::dom::parser p; walk(p.parse(dc.text).value(), dd); - const bool ok = a.nodes == b.nodes && a.nodes == c.nodes && a.nodes == dd.nodes && a.str == b.str && a.str == c.str && a.str == dd.str + const bool ok = a.nodes == b.nodes && a.nodes == e.nodes && a.str == e.str && std::fabs(a.num - e.num) <= 1e-9 * std::fabs(a.num) && a.nodes == c.nodes && a.nodes == dd.nodes && a.str == b.str && a.str == c.str && a.str == dd.str && std::fabs(a.num - b.num) <= 1e-9 * std::fabs(a.num) && std::fabs(a.num - c.num) <= 1e-9 * std::fabs(a.num); - const double pa = pick(dc.name, json::parse(dc.text)), pb = pick(dc.name, d.root()), pc = pick(dc.name, yyjson_doc_get_root(y)), pd = pick(dc.name, p.parse(dc.text).value()); - std::printf("check %-13s traverse %s select %s\n", dc.name.c_str(), ok ? "OK" : "MISMATCH", (pa == pb && pa == pc && pa == pd) ? "OK" : "MISMATCH"); + const double pa = pick(dc.name, json::parse(dc.text)), pb = pick(dc.name, d.root()), pe = pick(dc.name, ed.root()), pc = pick(dc.name, yyjson_doc_get_root(y)), pd = pick(dc.name, p.parse(dc.text).value()); + std::printf("check %-13s traverse %s select %s\n", dc.name.c_str(), ok ? "OK" : "MISMATCH", (pa == pb && pa == pe && pa == pc && pa == pd) ? "OK" : "MISMATCH"); yyjson_doc_free(y); + + // the dumps of the engines may format floats differently, so only: every dump is non-empty, and the + // dump of json_view (also editable) and of json::parse parse back to the same number of nodes + const std::string dj = json::parse(dc.text).dump(); + const std::string dv = d.root().dump(); + const std::string de = ed.root().dump(); + std::size_t yn = 0; + yyjson_doc* y2 = yyjson_read(dc.text.data(), dc.text.size(), 0); + char* yo = yyjson_write(y2, 0, &yn); + const bool dumps_ok = !dj.empty() && !dv.empty() && !de.empty() && yo != nullptr && yn > 0 && !simdjson::to_string(p.parse(dc.text).value()).empty(); + std::free(yo); + yyjson_doc_free(y2); + stats sj_nodes, sv_nodes, se_nodes; + walk(json::parse(dj), sj_nodes); + walk(json::parse(dv), sv_nodes); + walk(json::parse(de), se_nodes); + std::printf("check %-13s dump %s\n", dc.name.c_str(), (dumps_ok && sj_nodes.nodes == a.nodes && sv_nodes.nodes == a.nodes && se_nodes.nodes == a.nodes) ? "OK" : "MISMATCH"); } std::FILE* csv = std::fopen("bench_view.csv", "w"); @@ -669,6 +718,7 @@ int main(int argc, char** argv) workloads.push_back({"parse", { {"json_view", [&] { auto d = json_document::parse(s); g_sink = static_cast(d.node_count()); }}, {"json_view (reused)", [&] { reused.read(s); g_sink = static_cast(reused.node_count()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); g_sink = static_cast(d.root().size()); }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); g_sink = static_cast(yyjson_doc_get_val_count(d)); yyjson_doc_free(d); }}, {"simdjson DOM", [&] { auto e = sj.parse(ps).value_unsafe(); g_sink = e.is_object(); }}, {"simdjson DOM (fresh)", [&] { simdjson::dom::parser p; auto e = p.parse(ps).value_unsafe(); g_sink = e.is_object(); }}, @@ -680,6 +730,7 @@ int main(int argc, char** argv) workloads.push_back({"traverse", { {"json_view", [&] { auto d = json_document::parse(s); stats st; walk(d.root(), st); g_sink = st.num; }}, {"json_view (reused)", [&] { reused.read(s); stats st; walk(reused.root(), st); g_sink = st.num; }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); stats st; walk(d.root(), st); g_sink = st.num; }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); stats st; walk(yyjson_doc_get_root(d), st); g_sink = st.num; yyjson_doc_free(d); }}, {"simdjson DOM", [&] { stats st; walk(sj.parse(ps).value_unsafe(), st); g_sink = st.num; }}, {"simdjson OD", [&] { auto d = od.iterate(ps).value_unsafe(); stats st; walk_od(d.get_value().value_unsafe(), st); g_sink = st.num; }}, @@ -691,15 +742,31 @@ int main(int argc, char** argv) workloads.push_back({"select", { {"json_view", [&] { auto d = json_document::parse(s); g_sink = pick(name, d.root()); }}, {"json_view (reused)", [&] { reused.read(s); g_sink = pick(name, reused.root()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); g_sink = pick(name, d.root()); }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); g_sink = pick(name, yyjson_doc_get_root(d)); yyjson_doc_free(d); }}, {"simdjson DOM", [&] { g_sink = pick(name, sj.parse(ps).value_unsafe()); }}, {"simdjson OD", [&] { auto d = od.iterate(ps).value_unsafe(); g_sink = pick_od(name, d); }}, {"json::parse", [&] { json j = json::parse(s); g_sink = pick(name, j); }}, }}); + // pipeline: a fresh document, then a dump of it, in one timed call + // (simdjson DOM reuses its parser, as in the parse row); the traverse + // workload above is already parse+traverse + workloads.push_back({"parse+dump", { + {"json_view", [&] { auto d = json_document::parse(s); std::string o = d.root().dump(); g_sink = static_cast(o.size()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); std::string o = d.root().dump(); g_sink = static_cast(o.size()); }}, + {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); std::size_t n = 0; char* o = yyjson_write(d, 0, &n); g_sink = static_cast(n); std::free(o); yyjson_doc_free(d); }}, + {"simdjson DOM", [&] { std::string o = simdjson::to_string(sj.parse(ps).value_unsafe()); g_sink = static_cast(o.size()); }}, +#if JSON_VIEW_BENCH_BOOST + {"Boost.JSON", [&] { boost::json::monotonic_resource mr; auto v = boost::json::parse(s, &mr); std::string o = boost::json::serialize(v); g_sink = static_cast(o.size()); }}, +#endif + {"json::parse", [&] { json j = json::parse(s); std::string o = j.dump(); g_sink = static_cast(o.size()); }}, + }}); { // serialization of an already parsed document static json_document vd; vd.read(s); + static json_editable_document ed = json_editable_document::parse(s); + ed = json_editable_document::parse(s); static yyjson_doc* yd = nullptr; if (yd) { @@ -713,6 +780,7 @@ int main(int argc, char** argv) jd = json::parse(s); workloads.push_back({"dump", { {"json_view", [&] { std::string o = vd.root().dump(); g_sink = static_cast(o.size()); }}, + {"json_view (editable)", [&] { std::string o = ed.root().dump(); g_sink = static_cast(o.size()); }}, {"yyjson", [&] { std::size_t n = 0; char* o = yyjson_write(yd, 0, &n); g_sink = static_cast(n); std::free(o); }}, {"simdjson DOM", [&] { std::string o = simdjson::to_string(se); g_sink = static_cast(o.size()); }}, {"json::parse", [&] { std::string o = jd.dump(); g_sink = static_cast(o.size()); }}, @@ -740,7 +808,7 @@ int main(int argc, char** argv) } } const double ref = best[0]; - std::printf("%-13s %-9s", dc.name.c_str(), wl.first.c_str()); + std::printf("%-13s %-14s", dc.name.c_str(), wl.first.c_str()); for (std::size_t k = 0; k < wl.second.size(); ++k) { const double us = best[k] / 1e3; diff --git a/tests/benchmarks/json_view/compare.py b/tests/benchmarks/json_view/compare.py index 6573c88af..1e8ae2494 100755 --- a/tests/benchmarks/json_view/compare.py +++ b/tests/benchmarks/json_view/compare.py @@ -239,6 +239,8 @@ def main(): ap.add_argument('--download', action='store_true', help='use pinned downloads instead of system libraries') ap.add_argument('--no-boost', action='store_true', help='skip Boost.JSON') ap.add_argument('--native', action='store_true', help='compile for this CPU (-march=native / -mcpu=native)') + ap.add_argument('--aligned', action='store_true', + help='also build and run with all functions (and loops) aligned, to make code layout artifacts visible') ap.add_argument('--rounds', type=int, default=30, help='rounds of bench_view (default: 30)') ap.add_argument('--corpus', nargs='*', default=[], help='more files for bench_corpus') ap.add_argument('--build-dir', default=os.path.join(HERE, 'build'), help='where to build (default: build/ next to this script)') @@ -282,32 +284,46 @@ def main(): + ['-I' + d for d in lib.include]) objects.append(obj) - binaries = {} - for bench in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']: - exe = os.path.join(args.build_dir, bench) - run(cxx + flags + include + [os.path.join(HERE, bench + '.cpp')] + objects + link + ['-o', exe]) - binaries[bench] = exe + cxx_version = output(cxx + ['--version']) + is_clang = 'clang' in cxx_version.lower() + # alignment flags: functions and (gcc) loops on 64-byte boundaries + align_flags = ['-mllvm', '-align-all-functions=6'] if is_clang else ['-falign-functions=64', '-falign-loops=64'] + + # the builds: the default one and, with --aligned, one with aligned code + variants = [('default build', args.build_dir, [])] + if args.aligned: + variants.append(('aligned build', os.path.join(args.build_dir, 'aligned'), align_flags)) - # run: bench_view on its documents, bench_corpus on those and the given files corpus = [os.path.join(args.data, f) for f in DEFAULT_CORPUS] + args.corpus - outputs = {} - outputs['bench_view'] = run([binaries['bench_view'], args.data, str(args.rounds)], cwd=args.build_dir, - capture_output=True, text=True).stdout - outputs['bench_corpus'] = run([binaries['bench_corpus']] + corpus, cwd=args.build_dir, - capture_output=True, text=True).stdout - outputs['bench_edit'] = run([binaries['bench_edit'], args.data, str(max(1, args.rounds // 2))], cwd=args.build_dir, - capture_output=True, text=True).stdout - outputs['bench_decode'] = run([binaries['bench_decode'], args.data, str(max(1, args.rounds // 2))], cwd=args.build_dir, - capture_output=True, text=True).stdout - for name, text in outputs.items(): - print(text) + # label -> {benchmark: output} + all_outputs = {} + for label, build_dir, extra in variants: + os.makedirs(build_dir, exist_ok=True) + binaries = {} + for bench in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']: + exe = os.path.join(build_dir, bench) + run(cxx + flags + extra + include + [os.path.join(HERE, bench + '.cpp')] + objects + link + ['-o', exe]) + binaries[bench] = exe + + # run: bench_view on its documents, bench_corpus on those and the given files + outputs = {} + outputs['bench_view'] = run([binaries['bench_view'], args.data, str(args.rounds)], cwd=build_dir, + capture_output=True, text=True).stdout + outputs['bench_corpus'] = run([binaries['bench_corpus']] + corpus, cwd=build_dir, + capture_output=True, text=True).stdout + outputs['bench_edit'] = run([binaries['bench_edit'], args.data, str(max(1, args.rounds // 2))], cwd=build_dir, + capture_output=True, text=True).stdout + outputs['bench_decode'] = run([binaries['bench_decode'], args.data, str(max(1, args.rounds // 2))], + cwd=build_dir, capture_output=True, text=True).stdout + for name, text in outputs.items(): + print(text) + all_outputs[label] = outputs # results with their metadata now = datetime.datetime.now() host = re.sub(r'[^A-Za-z0-9-]+', '-', platform.node().split('.')[0]) or 'host' stem = os.path.join(HERE, 'results', f'{now:%Y-%m-%d}-{host}') os.makedirs(os.path.dirname(stem), exist_ok=True) - cxx_version = output(cxx + ['--version']) compiler_version = cxx_version.splitlines()[0] if cxx_version else ' '.join(cxx) meta = [ ('date', f'{now:%Y-%m-%d %H:%M}'), @@ -316,6 +332,7 @@ def main(): ('OS', f'{platform.system()} {platform.release()} ({platform.machine()})'), ('compiler', compiler_version), ('flags', ' '.join(flags)), + ('aligned results', ('included (flags: ' + ' '.join(align_flags) + ')') if args.aligned else 'not included'), ('yyjson', libs['yyjson'].version), ('simdjson', libs['simdjson'].version), ('Boost.JSON', libs['boost'].version if with_boost else 'skipped (not found)'), @@ -328,15 +345,20 @@ def main(): f.write('| | |\n|---|---|\n') for key, value in meta: f.write(f'| {key} | {value} |\n') - for name, text in outputs.items(): - f.write(f'\n## {name}\n\n```\n{text.rstrip()}\n```\n') + for label, outputs in all_outputs.items(): + # (without --aligned there is only the default build, and no extra heading level) + prefix = f'{label}: ' if args.aligned else '' + for name, text in outputs.items(): + f.write(f'\n## {prefix}{name}\n\n```\n{text.rstrip()}\n```\n') with open(stem + '.csv', 'w', encoding='utf-8') as out: out.write(''.join(f'# {key}: {value}\n' for key, value in meta)) - for name in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']: - path = os.path.join(args.build_dir, name + '.csv') - if os.path.isfile(path): - with open(path, encoding='utf-8') as f: - out.write(f'# {name}\n' + f.read()) + for label, build_dir, _ in variants: + for name in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']: + path = os.path.join(build_dir, name + '.csv') + if os.path.isfile(path): + with open(path, encoding='utf-8') as f: + # (the default build keeps its plain section names) + out.write(f'# {name}' + (f' ({label})' if args.aligned else '') + '\n' + f.read()) print(f'results: {stem}.md, {stem}.csv') diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index dd78f7ff4..090ab4b41 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -3030,3 +3030,144 @@ TEST_CASE("a json_sax_dom_parser keeps the values read before a parse error") CHECK(read_before_error(c.first) == c.second); } } + +TEST_CASE("object keys are delivered decoded and validated") +{ + // json_sax::key and the key event of a parser callback see the decoded name, and + // only after the whole name was scanned and validated + const auto keys_of_format = [](const std::string & text, const json::input_format_t format) + { + SaxEventLogger l; + json::sax_parse(text, &l, format); + std::vector keys; + for (const auto& e : l.events) + { + if (e.compare(0, 4, "key(") == 0) + { + keys.push_back(e.substr(4, e.size() - 5)); + } + } + return keys; + }; + + const auto keys_of = [&keys_of_format](const std::string & text) + { + return keys_of_format(text, json::input_format_t::json); + }; + + const auto callback_keys_of = [](const std::string & text) + { + std::vector keys; + const json::parser_callback_t cb = [&keys](int /*depth*/, json::parse_event_t event, json & parsed) + { + if (event == json::parse_event_t::key) + { + keys.push_back(parsed.get()); + } + return true; + }; + const json parsed = json::parse(text, cb, false); + static_cast(parsed); + return keys; + }; + + SECTION("different spellings of a name yield the same string") + { + const std::string e_acute = "\xc3\xa9"; + const std::string text = std::string(R"({"/":1,"\/":2,"\u002f":3,"\u00e9":4,"\u00E9":5,")") + e_acute + R"(":6})"; + const std::vector expected = {"/", "/", "/", e_acute, e_acute, e_acute}; + CHECK(keys_of(text) == expected); + CHECK(callback_keys_of(text) == expected); + } + + SECTION("escape sequences are decoded") + { + const std::string simple = R"({"\"\\\/\b\f\n\r\t":1})"; + const std::vector simple_expected = {std::string("\"\\/\b\f\n\r\t")}; + CHECK(keys_of(simple) == simple_expected); + CHECK(callback_keys_of(simple) == simple_expected); + + const std::string nul = R"({"\u0000":1})"; + const std::vector nul_expected = {std::string(1, '\0')}; + CHECK(keys_of(nul) == nul_expected); + CHECK(callback_keys_of(nul) == nul_expected); + + const std::string pair = R"({"\ud83d\ude00":1})"; + const std::vector pair_expected = {"\xf0\x9f\x98\x80"}; + CHECK(keys_of(pair) == pair_expected); + CHECK(callback_keys_of(pair) == pair_expected); + } + + SECTION("no normalization and no case conversion") + { + const std::string text = std::string("{\"\xc3\xa9\":1,\"e\xcc\x81\":2,\"A\":3,\"a\":4}"); + const std::vector expected = {"\xc3\xa9", "e\xcc\x81", "A", "a"}; + CHECK(keys_of(text) == expected); + CHECK(callback_keys_of(text) == expected); + } + + SECTION("a name that is not valid is not delivered") + { + const std::vector texts = + { + R"({"\u12G4":1})", // malformed escape + R"({"\udc00":1})", // lone low surrogate + R"({"\ud800":1})", // lone high surrogate + R"({"\ud800\u0041":1})", // high surrogate not followed by a low one + R"({"\x":1})", // unknown escape + std::string("{\"a\x01\":1}"), // control character + std::string("{\"\xff\":1}") // ill-formed UTF-8 + }; + for (const auto& text : texts) + { + CAPTURE(text) + CHECK(!json::accept(text)); + CHECK(keys_of(text).empty()); + CHECK(callback_keys_of(text).empty()); + } + } + + SECTION("a name is delivered before the following colon is read") + { + const std::string text = R"({"a" 1})"; + const std::vector expected = {"a"}; + CHECK(!json::accept(text)); + CHECK(keys_of(text) == expected); + CHECK(callback_keys_of(text) == expected); + } + + SECTION("a repeated name is delivered once per occurrence") + { + const std::string text = R"({"a":1,"a":2})"; + const std::vector expected = {"a", "a"}; + CHECK(keys_of(text) == expected); + CHECK(callback_keys_of(text) == expected); + } + + SECTION("binary formats pass the stored bytes through") + { + const std::string stored = "\xff\\"; + const std::vector expected = {stored}; + const auto keys_of_bytes = [&keys_of_format](const std::vector& v, json::input_format_t format) + { + std::string text; + for (const auto byte : v) + { + text.push_back(static_cast(byte)); + } + return keys_of_format(text, format); + }; + + CHECK(keys_of_bytes({0xA1, 0x62, 0xFF, 0x5C, 0x01}, json::input_format_t::cbor) == expected); + CHECK(keys_of_bytes({0x81, 0xA2, 0xFF, 0x5C, 0x01}, json::input_format_t::msgpack) == expected); + CHECK(keys_of_bytes({'{', 'i', 2, 0xFF, 0x5C, 'i', 1, '}'}, json::input_format_t::ubjson) == expected); + CHECK(keys_of_bytes({0x0D, 0, 0, 0, 0x10, 0xFF, 0x5C, 0, 1, 0, 0, 0, 0}, json::input_format_t::bson) == expected); + } + + SECTION("BON8 rejects ill-formed UTF-8 in a name") + { + const std::vector valid = {"\\"}; + CHECK(keys_of_format(std::string("\x87\x5C\x91"), json::input_format_t::bon8) == valid); + CHECK(keys_of_format(std::string("\x87\xC3\x28\x91"), json::input_format_t::bon8).empty()); + } +} diff --git a/tests/src/unit-json_view_compare.cpp b/tests/src/unit-json_view_compare.cpp new file mode 100644 index 000000000..48527adb4 --- /dev/null +++ b/tests/src/unit-json_view_compare.cpp @@ -0,0 +1,364 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; +using nlohmann::json_document; +using nlohmann::ordered_json_editable_document; +using nlohmann::json_view; +using nlohmann::ordered_json_document; + +#include "json_view_test_helpers.hpp" +using json_view_test::generator; + +#include +#include +#include +#include + +// These tests were split off unit-json_view_dump.cpp, whose object file got too +// large for the MinGW assembler ("file too big"). Like that file, this one is +// also built with C++17, as it mentions JSON_HAS_CPP_17. + +TEST_CASE("json_view comparison") +{ + SECTION("equality of the values parse() produces") + { + generator g; + std::vector texts; + for (int i = 0; i < 600; ++i) + { + std::string text; + g.value(text, 0); + texts.push_back(text); + // the same value written differently: sorted keys, canonical numbers + texts.push_back(json::parse(text).dump(1)); + } + for (std::size_t i = 0; i + 2 < texts.size(); ++i) + { + for (std::size_t k = i; k < i + 3; ++k) + { + CAPTURE(texts[i]) + CAPTURE(texts[k]) + const json_document a = json_document::parse(texts[i]); + const json_document b = json_document::parse(texts[k]); + const json ja = json::parse(texts[i]); + const json jb = json::parse(texts[k]); + CHECK((a.root() == b.root()) == (ja == jb)); + CHECK((a.root() != b.root()) == (ja != jb)); + CHECK((a.root() == jb) == (ja == jb)); + CHECK((jb == a.root()) == (ja == jb)); + CHECK((a.root() != jb) == (ja != jb)); + CHECK((jb != a.root()) == (ja != jb)); + + // ordered_json compares members in order + const ordered_json_document oa = ordered_json_document::parse(texts[i]); + const ordered_json_document ob = ordered_json_document::parse(texts[k]); + const ordered_json oja = ordered_json::parse(texts[i]); + const ordered_json ojb = ordered_json::parse(texts[k]); + CHECK((oa.root() == ob.root()) == (oja == ojb)); + CHECK((oa.root() == ojb) == (oja == ojb)); + } + } + } + + SECTION("objects whose first keys differ or repeat") + { + const std::array texts = + { + { + R"({"a":1,"b":2})", R"({"a":1,"c":2})", R"({"b":1,"a":2})", R"({"a":1,"a":2,"b":3})", R"({"a":2,"b":3})", + R"({"a":1,"b":1,"a":2})", R"({"a":2,"b":1})", R"({"a":1,"a":2})", R"({"b":1})", R"({"a":1,"b":2,"c":3,"d":4,"e":5})", + R"({"a":1,"b":2,"c":3,"d":4,"f":5})", R"({"a":1,"b":2,"c":3,"a":4,"e":5})", R"({"a":4,"b":2,"c":3,"e":5})", + R"({"\u0061":1,"b":2})", R"({})", R"([{"a":1,"b":2}])", R"([{"a":1,"c":2}])" + } + }; + for (const char* const x : texts) + { + for (const char* const y : texts) + { + CAPTURE(x) + CAPTURE(y) + const ordered_json_document dx = ordered_json_document::parse(x); + const ordered_json_document dy = ordered_json_document::parse(y); + const ordered_json_editable_document ex = ordered_json_editable_document::parse(x); + const ordered_json jx = ordered_json::parse(x); + const ordered_json jy = ordered_json::parse(y); + CHECK((dx.root() == dy.root()) == (jx == jy)); + CHECK((dx.root() == jy) == (jx == jy)); + CHECK((ex.root() == dy.root()) == (jx == jy)); + } + } + } + + SECTION("objects with duplicate keys, small and large") + { + // n members with keys in descending order; with duplicates, some keys + // repeat later (the first after unique ones in key order); in reverse, + // the members are in ascending key order + const auto object = [](int n, bool duplicates, bool reverse, int value) + { + std::string s = "{"; + for (int i = 0; i < n; ++i) + { + const int k = reverse ? i : n - 1 - i; + s += (i != 0 ? ",\"k" : "\"k") + std::to_string(100 + k) + "\":" + std::to_string(value + k); + } + if (duplicates) + { + for (int k = n / 2; k < n; k += 3) + { + s += ",\"k" + std::to_string(100 + k) + "\":" + std::to_string(value + k); + } + } + return s + "}"; + }; + std::vector texts; + for (const int n : + { + 3, 16, 17, 40 + }) + { + for (const int flags : + { + 0, 1, 2, 3 + }) + { + texts.push_back(object(n, (flags & 1) != 0, (flags & 2) != 0, 0)); + } + texts.push_back(object(n, true, false, 1)); // other values + } + // many objects in one comparison (working memory reused across objects) + std::string array = "["; + std::string array_other = "["; + for (std::size_t i = 0; i < texts.size(); ++i) + { + array += (i != 0 ? "," : "") + texts[i]; + array_other += (i != 0 ? "," : "") + texts[texts.size() - 1 - i]; + } + texts.push_back(array + "]"); + texts.push_back(array_other + "]"); + for (const std::string& x : texts) + { + for (const std::string& y : texts) + { + CAPTURE(x) + CAPTURE(y) + const ordered_json_document dx = ordered_json_document::parse(x); + const ordered_json_document dy = ordered_json_document::parse(y); + const ordered_json jx = ordered_json::parse(x); + const ordered_json jy = ordered_json::parse(y); + CHECK((dx.root() == dy.root()) == (jx == jy)); + CHECK((dx.root() == jy) == (jx == jy)); + const json_document ux = json_document::parse(x); + const json uy = json::parse(y); + CHECK((ux.root() == uy) == (json::parse(x) == uy)); + } + } + } + + SECTION("numbers, duplicate keys, member order") + { + const auto same = [](const char* x, const char* y) + { + const json_document dx = json_document::parse(x); + const json_document dy = json_document::parse(y); + return dx.root() == dy.root(); + }; + CHECK(same("1", "1.0")); + CHECK(same("[1, -1, 2.5]", "[1.0, -1.0, 25e-1]")); + CHECK(!same("1", "1.5")); + CHECK(same("18446744073709551615", "18446744073709551615")); + CHECK(same(R"({"a": 1, "a": 2})", R"({"a": 2})")); + CHECK(!same(R"({"a": 1, "a": 2})", R"({"a": 1})")); + CHECK(same(R"({"a": 1, "b": 2})", R"({"b": 2, "a": 1})")); + CHECK(!same(R"({"a": 1})", R"({"a": 1, "b": 2})")); + CHECK(!same("[1, 2]", "[2, 1]")); + CHECK(!same("\"a\"", "\"b\"")); + CHECK(same("\"\\u00e9\"", "\"\xc3\xa9\"")); + CHECK(!same("null", "false")); + CHECK(!same("[]", "{}")); + const ordered_json_document dup = ordered_json_document::parse(R"({"a": 1, "b": 2, "a": 3})"); + const ordered_json_document last = ordered_json_document::parse(R"({"a": 3, "b": 2})"); + CHECK(dup.root() == last.root()); + // (== compares as parse() resolves duplicates, a lookup finds the first) + CHECK(dup.root()["a"].materialize() == 1); + CHECK(dup.root()["a"] != last.root()["a"]); + const ordered_json_document ab = ordered_json_document::parse(R"({"a": 1, "b": 2})"); + const ordered_json_document ba = ordered_json_document::parse(R"({"b": 2, "a": 1})"); + CHECK(ab.root() != ba.root()); + + // discarded values compare as basic_json's do + const json discarded(json::value_t::discarded); + CHECK((json_view() == json_view()) == (discarded == discarded)); // NOLINT(readability-container-size-empty): operator== is tested + CHECK((json_view() == discarded) == (discarded == discarded)); // NOLINT(readability-container-size-empty) + const json_document null_document = json_document::parse("null"); + CHECK(!(json_view() == null_document.root())); // NOLINT(readability-container-size-empty) + CHECK(!(null_document.root() == discarded)); + } + + SECTION("values that are the same text, or the same value written differently, at any depth") + { + // nested arrays and objects around depths where the walk over equal text gives up + for (const int depth : + { + 1, 5, 47, 48, 49, 50, 60, 300 + }) + { + CAPTURE(depth) + std::string head; + std::string tail; + for (int i = 0; i < depth; ++i) + { + head += i % 2 == 0 ? R"({"k":[1,"s",{"x":true},)" : R"([null,2.5,{"y":)"; + tail.insert(0, i % 2 == 0 ? "]}" : "}]"); + } + // the same text, one number changed, and one number written differently + const auto with_number = [&](const char* number) + { + std::string result = head; + result += number; + result += tail; + return result; + }; + const std::string text = with_number("7"); + const std::string changed = with_number("8"); + const std::string spelled = with_number("7.0"); + const json_document a = json_document::parse(text); + const json_document b = json_document::parse(text); + const json_document c = json_document::parse(changed); + const json_document d = json_document::parse(spelled); + CHECK(a.root() == b.root()); + CHECK(a.root() != c.root()); + CHECK(c.root() != a.root()); + CHECK(a.root() == d.root()); + CHECK(d.root() == a.root()); + CHECK(a.root() == json::parse(text)); + CHECK(a.root() != json::parse(changed)); + CHECK(json::parse(spelled) == a.root()); + const nlohmann::json_editable_document e = nlohmann::json_editable_document::parse(text); + CHECK(a.root() == e.root()); + CHECK(e.root() == a.root()); + CHECK(c.root() != e.root()); + CHECK(e.root() == e.root()); + const ordered_json_document oa = ordered_json_document::parse(text); + const ordered_json_document oc = ordered_json_document::parse(changed); + CHECK(oa.root() != oc.root()); + CHECK(oa.root() == ordered_json::parse(text)); + CHECK(oc.root() != ordered_json::parse(text)); + } + + // not the same text, but the same value + const auto same = [](const char* x, const char* y) + { + const json_document dx = json_document::parse(x); + const json_document dy = json_document::parse(y); + const bool equal = dx.root() == dy.root(); + CHECK(equal == (json::parse(x) == json::parse(y))); + CHECK(equal == (dx.root() == json::parse(y))); + return equal; + }; + CHECK(same(R"({"a":1,"a":2})", R"({"a":3,"a":2})")); + CHECK(same(R"({"a":1,"b":[1,{"c":2,"c":3}]})", R"({"b":[1,{"c":9,"c":3}],"a":1})")); + CHECK(!same(R"({"a":1,"a":2})", R"({"a":2,"a":3})")); + CHECK(same("[1.0,-0,0,1e2]", "[1.00,0,-0,100.0]")); + CHECK(same(R"(["\u00e9","\n"])", "[\"\xc3\xa9\",\"\\n\"]")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"b":false}]})")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"c":null}]})")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"b":null,"c":1}]})")); + + // edited documents: moved sequences, links, and values of new nodes + nlohmann::json_editable_document edited = nlohmann::json_editable_document::parse(R"({"a":[1,2,3],"b":{"c":"d"},"e":0.5})"); + const json_document plain = json_document::parse(R"({"a":[1,2,3,4],"b":{"c":"x","y":null},"e":0.5,"z":[]})"); + edited.push_back(edited.root()["a"], 4); + edited.set(edited.root()["b"], "c", "x"); + edited.set(edited.root()["b"], "y", nullptr); + edited.set(edited.root(), "z", json::array()); + CHECK(edited.root() == plain.root()); + CHECK(plain.root() == edited.root()); + edited.erase(edited.root()["a"], 0); + CHECK(edited.root() != plain.root()); + CHECK(edited.root()["a"] == json::parse("[2,3,4]")); + const json_document tail = json_document::parse("[2,3,4]"); + CHECK(edited.root()["a"] == tail.root()); + } + + SECTION("large documents, with a difference at any place") + { + const auto make = [](std::size_t count, std::size_t changed, bool swap_members) + { + std::string text = R"({"b":1,"list":[)"; + for (std::size_t i = 0; i < count; ++i) + { + text += (i != 0 ? "," : ""); + const std::string id = std::to_string(i == changed ? i + 1 : i); + if (swap_members && i % 3 == 0) + { + text += R"({"v":[1.5,"s",{"k":)"; + text += id; + text += R"(}],"id":)"; + text += id; + text += "}"; + } + else + { + text += R"({"id":)"; + text += id; + text += R"(,"v":[1.5,"s",{"k":)"; + text += id; + text += "}]}"; + } + } + return text + R"(],"a":{"y":2,"x":3}})"; + }; + const std::size_t count = 3000; + for (const std::size_t changed : + { + count, std::size_t{0}, std::size_t{1}, count / 2, count - 1 + }) + { + for (const bool swap : + { + false, true + }) + { + CAPTURE(changed) + CAPTURE(swap) + const std::string text_a = make(count, count, false); + const std::string text_b = make(count, changed, swap); + const json_document a = json_document::parse(text_a); + const json_document b = json_document::parse(text_b); + const bool equal = changed == count; + CHECK((a.root() == b.root()) == equal); + CHECK((b.root() == a.root()) == equal); + CHECK((a.root() == json::parse(text_b)) == equal); + CHECK((json::parse(text_b) == a.root()) == equal); + CHECK((a.root()["list"] == b.root()["list"]) == equal); + const ordered_json_document oa = ordered_json_document::parse(text_a); + const ordered_json_document ob = ordered_json_document::parse(text_b); + CHECK((oa.root() == ob.root()) == (equal && !swap)); + CHECK((oa.root() == ordered_json::parse(text_b)) == (equal && !swap)); + } + } + } + + SECTION("deep nesting") + { + const std::string deep = std::string(100000, '[') + std::string(100000, ']'); + const json_document a = json_document::parse(deep); + const json_document b = json_document::parse(deep); + CHECK(a.root() == b.root()); + CHECK(a.root() == json::parse(deep)); + const std::string other = std::string(100000, '[') + "1" + std::string(100000, ']'); + const json_document c = json_document::parse(other); + CHECK(a.root() != c.root()); + } +} diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index f3ccc004e..6470ab680 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -432,101 +432,6 @@ TEST_CASE("json_view dump") } } -TEST_CASE("json_view comparison") -{ - SECTION("equality of the values parse() produces") - { - generator g; - std::vector texts; - for (int i = 0; i < 600; ++i) - { - std::string text; - g.value(text, 0); - texts.push_back(text); - // the same value written differently: sorted keys, canonical numbers - texts.push_back(json::parse(text).dump(1)); - } - for (std::size_t i = 0; i + 2 < texts.size(); ++i) - { - for (std::size_t k = i; k < i + 3; ++k) - { - CAPTURE(texts[i]) - CAPTURE(texts[k]) - const json_document a = json_document::parse(texts[i]); - const json_document b = json_document::parse(texts[k]); - const json ja = json::parse(texts[i]); - const json jb = json::parse(texts[k]); - CHECK((a.root() == b.root()) == (ja == jb)); - CHECK((a.root() != b.root()) == (ja != jb)); - CHECK((a.root() == jb) == (ja == jb)); - CHECK((jb == a.root()) == (ja == jb)); - CHECK((a.root() != jb) == (ja != jb)); - CHECK((jb != a.root()) == (ja != jb)); - - // ordered_json compares members in order - const ordered_json_document oa = ordered_json_document::parse(texts[i]); - const ordered_json_document ob = ordered_json_document::parse(texts[k]); - const ordered_json oja = ordered_json::parse(texts[i]); - const ordered_json ojb = ordered_json::parse(texts[k]); - CHECK((oa.root() == ob.root()) == (oja == ojb)); - CHECK((oa.root() == ojb) == (oja == ojb)); - } - } - } - - SECTION("numbers, duplicate keys, member order") - { - const auto same = [](const char* x, const char* y) - { - const json_document dx = json_document::parse(x); - const json_document dy = json_document::parse(y); - return dx.root() == dy.root(); - }; - CHECK(same("1", "1.0")); - CHECK(same("[1, -1, 2.5]", "[1.0, -1.0, 25e-1]")); - CHECK(!same("1", "1.5")); - CHECK(same("18446744073709551615", "18446744073709551615")); - CHECK(same(R"({"a": 1, "a": 2})", R"({"a": 2})")); - CHECK(!same(R"({"a": 1, "a": 2})", R"({"a": 1})")); - CHECK(same(R"({"a": 1, "b": 2})", R"({"b": 2, "a": 1})")); - CHECK(!same(R"({"a": 1})", R"({"a": 1, "b": 2})")); - CHECK(!same("[1, 2]", "[2, 1]")); - CHECK(!same("\"a\"", "\"b\"")); - CHECK(same("\"\\u00e9\"", "\"\xc3\xa9\"")); - CHECK(!same("null", "false")); - CHECK(!same("[]", "{}")); - const ordered_json_document dup = ordered_json_document::parse(R"({"a": 1, "b": 2, "a": 3})"); - const ordered_json_document last = ordered_json_document::parse(R"({"a": 3, "b": 2})"); - CHECK(dup.root() == last.root()); - // (== compares as parse() resolves duplicates, a lookup finds the first) - CHECK(dup.root()["a"].materialize() == 1); - CHECK(dup.root()["a"] != last.root()["a"]); - const ordered_json_document ab = ordered_json_document::parse(R"({"a": 1, "b": 2})"); - const ordered_json_document ba = ordered_json_document::parse(R"({"b": 2, "a": 1})"); - CHECK(ab.root() != ba.root()); - - // discarded values compare as basic_json's do - const json discarded(json::value_t::discarded); - CHECK((json_view() == json_view()) == (discarded == discarded)); // NOLINT(readability-container-size-empty): operator== is tested - CHECK((json_view() == discarded) == (discarded == discarded)); // NOLINT(readability-container-size-empty) - const json_document null_document = json_document::parse("null"); - CHECK(!(json_view() == null_document.root())); // NOLINT(readability-container-size-empty) - CHECK(!(null_document.root() == discarded)); - } - - SECTION("deep nesting") - { - const std::string deep = std::string(100000, '[') + std::string(100000, ']'); - const json_document a = json_document::parse(deep); - const json_document b = json_document::parse(deep); - CHECK(a.root() == b.root()); - CHECK(a.root() == json::parse(deep)); - const std::string other = std::string(100000, '[') + "1" + std::string(100000, ']'); - const json_document c = json_document::parse(other); - CHECK(a.root() != c.root()); - } -} - TEST_CASE("json_view large objects") { // objects with 128 members or more are looked up with a hash index diff --git a/tests/src/unit-json_view_edit.cpp b/tests/src/unit-json_view_edit.cpp index 4a6fbe69d..1b1a24ec8 100644 --- a/tests/src/unit-json_view_edit.cpp +++ b/tests/src/unit-json_view_edit.cpp @@ -941,3 +941,123 @@ TEST_CASE("json_view edits: duplicate keys") CHECK(d.root()["o"]["d"].get() == 0); } } + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("json_view edits: check_utf8 agrees with the byte-wise check") +{ + namespace dv = nlohmann::detail::view; + + // "ok" or the exception message + const auto run = [](bool fast, const std::string & s) -> std::string + { + try + { + if (fast) + { + dv::check_utf8(s.data(), s.size()); + } + else + { + dv::check_utf8_bytewise(reinterpret_cast(s.data()), s.size(), 0); + } + } + catch (const json::type_error& e) + { + return {e.what()}; + } + return {"ok"}; + }; + std::size_t errors = 0; + const auto same = [&](const std::string & s) + { + const std::string a = run(true, s); + const std::string b = run(false, s); + if (a != b) + { + INFO("fast: " << a << ", byte-wise: " << b << ", length " << s.size()); + CHECK(a == b); + } + errors += b != "ok" ? 1u : 0u; + }; + + const std::vector pieces = + { + "a", "Z", " ", "\"", "\\", "\n", "\x01", "\x7f", "\xc3\xa4", "\xe4\xb8\xad", "\xf0\x9f\x98\x80", + "\x80", "\xbf", "\xc0\x80", "\xc1\xbf", "\xc2", "\xe0\x80\x80", "\xe0\x9f\xbf", "\xed\xa0\x80", "\xed\xbf\xbf", + "\xf0\x80\x80\x80", "\xf0\x8f\xbf\xbf", "\xf4\x90\x80\x80", "\xf4\x8f\xbf\xbf", "\xf5", "\xf8", "\xff", "\xfe", + "\xe4\xb8", "\xf0\x9f\x98", "\xf0\x9f", "\xf0" + }; + + // every piece behind prefixes of every length, and followed by more bytes + for (std::size_t prefix = 0; prefix <= 64; ++prefix) + { + for (const auto& piece : pieces) + { + same(std::string(prefix, 'x') + piece); + same(std::string(prefix, 'x') + piece + "yz"); + same(std::string(prefix, '"') + piece); + same(std::string(prefix, '\xc3') + piece); + same(std::string(prefix / 2, '\xc3').append(prefix / 2, '\xa4') + piece); + same(std::string(prefix, 'x') + "\xe4\xb8\xad" + piece); + } + } + // an invalid byte at every position of valid strings of every length + for (std::size_t len = 0; len <= 64; ++len) + { + const std::string ascii(len, 'a'); + std::string mixed; + while (mixed.size() < len) + { + if (mixed.size() % 7 == 0) + { + mixed += "\xc3\xa4"; + } + else if (mixed.size() % 11 == 0) + { + mixed += "\xf0\x9f\x98\x80"; + } + else + { + mixed += "a"; + } + } + same(ascii); + same(mixed); + for (std::size_t pos = 0; pos < len; ++pos) + { + for (const char bad : + {'\x80', '\xc0', '\xc2', '\xe0', '\xf4', '\xff', '"', '\\', '\x1f' + }) + { + std::string s = ascii; + s[pos] = bad; + same(s); + s = mixed; + s[pos] = bad; + same(s); + } + } + } + // random bytes, biased to the interesting ones + std::mt19937 gen(5869); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + static const std::array interesting = {{0x00, 0x1f, 0x22, 0x5c, 0x7f, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0xc0, 0xc2, 0xdf, 0xe0, 0xe1, 0xed, 0xef, 0xf0, 0xf1, 0xf4, 0xf5, 0xff}}; + for (int i = 0; i < 100000; ++i) + { + std::string s(gen() % 80, 'a'); + for (auto& c : s) + { + const unsigned r = gen() % 8; + if (r == 0) + { + c = static_cast(gen() & 0xff); + } + else if (r == 1) + { + c = static_cast(interesting[gen() % interesting.size()]); + } + } + same(s); + } + CHECK(errors > 1000); +} +#endif diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp index ff1713371..5e392559d 100644 --- a/tests/src/unit-json_view_image.cpp +++ b/tests/src/unit-json_view_image.cpp @@ -185,8 +185,10 @@ std::uint32_t rng() TEST_CASE("json_view images: round trips") { - static_assert(json_view_image_test::has_save::value && json_view_image_test::has_load::value, "save() and load() on a little-endian target"); - static_assert(json_view_image_test::has_save::value && json_view_image_test::has_load::value, "save() and load() on a little-endian target"); + static_assert(json_view_image_test::has_save::value, "save() on a little-endian target"); + static_assert(json_view_image_test::has_load::value, "load() on a little-endian target"); + static_assert(json_view_image_test::has_save::value, "save() on a little-endian target"); + static_assert(json_view_image_test::has_load::value, "load() on a little-endian target"); SECTION("small documents") { @@ -699,6 +701,64 @@ TEST_CASE("json_view images: ownership") #if !defined(JSON_NOEXCEPTION) TEST_CASE("json_view images: errors") { + SECTION("float tokens: a changed byte is accepted if the text is still a number with the digits the node records") + { + for (const char* const token : + { + "-12.375", "0.5", "123456.0", "-0.0", "9999999999999999999.5", "0.000000000000000000000001", "1.25e10", "5E-3" + }) + { + CAPTURE(token) + const std::string text = std::string("[") + token + "]"; + const json_document d = json_document::parse(text); + const std::vector image = d.save(); + const std::size_t at = text_at(image) + 1; // the first byte of the token + const std::uint16_t recorded = node_at(image, 1).extra; + CHECK(loaded_dump(image) == json::parse(text).dump()); + for (std::size_t pos = 0; token[pos] != '\0'; ++pos) + { + for (unsigned int byte = 0; byte < 256; ++byte) + { + std::vector changed = image; + changed[at + pos] = static_cast(byte); + std::string t = token; + t[pos] = static_cast(byte); + // the digits of the integer part and of the fraction of the changed token + std::size_t integer = 0; + std::size_t fraction = 0; + std::size_t k = (t[0] == '-') ? 1 : 0; + for (; k < t.size() && t[k] >= '0' && t[k] <= '9'; ++k) + { + ++integer; + } + if (k < t.size() && t[k] == '.') + { + for (++k; k < t.size() && t[k] >= '0' && t[k] <= '9'; ++k) + { + ++fraction; + } + } + // (as a JSON text, the token may have white space around it) + const bool float_token = json::accept(t) && t.find_first_of(std::string(" \t\r\n\0", 5)) == std::string::npos && (t.find('.') != std::string::npos || t.find_first_of("eE") != std::string::npos) // NOLINT(abseil-string-find-str-contains) + && ((integer | (fraction << 8U)) == recorded); + CAPTURE(pos) + CAPTURE(byte) + std::string result; + try + { + result = loaded_dump(changed); + } + catch (const json::parse_error& e) + { + CHECK(std::string(e.what()) == check_failed); + result = "rejected"; + } + CHECK((result != "rejected") == float_token); + } + } + } + } + SECTION("invalid UTF-8 of a decoded string: the compact dump writes it, the others throw") { const json_document d = json_document::parse(R"({"k\n":"\u00e9x"})");