diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 4d8ee33e5..99010d81b 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index eb964c979..ba60d70e9 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -31,7 +31,7 @@ jobs: mkdir -p ./pr echo ${{ github.event.number }} > ./pr/number echo ${{ github.event.pull_request.user.login }} > ./pr/author - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: pr path: pr/ @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -170,7 +170,7 @@ jobs: # instead of installing the pinned astyle version locally. - name: Upload patch if: steps.diff.outputs.has_diff == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: amalgamation-patch path: patch/amalgamation.patch diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index 342c96bff..bf174c245 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 30 steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 9398fdbaf..43efd0e16 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -35,7 +35,7 @@ jobs: dry-run: false language: c++ - name: Upload Crash - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: failure() && steps.build.outcome == 'success' with: name: artifacts diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9e86353c6..1c4527f02 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -38,14 +38,14 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: languages: c-cpp # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/autobuild@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index 32bca8486..c01572a44 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -20,7 +20,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 2052c1d35..333546437 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index f37539a73..82d365c6d 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -31,7 +31,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -47,6 +47,6 @@ jobs: output: 'flawfinder_results.sarif' - name: Upload analysis results to GitHub Security tab - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: sarif_file: ${{github.workspace}}/flawfinder_results.sarif diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index a93c86ae2..21cf6c99c 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index 3256c819b..979f2937f 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index bbf11dd28..a50c1d2bb 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -72,7 +72,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: SARIF file path: results.sarif @@ -80,6 +80,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: sarif_file: results.sarif diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index d8c717e83..f9c11156d 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -65,7 +65,7 @@ jobs: # Upload SARIF file generated in previous step - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: sarif_file: semgrep.sarif if: always() diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1e690ccdd..10d527504 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index edce983a5..025f58179 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -25,7 +25,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -38,7 +38,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Check that Meson and pkg-config offer the CMake options run: make check_build_options - name: Install Meson @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -92,14 +92,14 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build run: cmake --build build --target ci_infer - name: Archive Infer report if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: infer-report path: ${{ github.workspace }}/build/build_infer/infer-out/report.txt @@ -112,7 +112,7 @@ jobs: target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind] steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -123,7 +123,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -142,7 +142,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -168,7 +168,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -178,7 +178,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -197,7 +197,7 @@ jobs: - name: Build run: cmake --build build --target ci_test_coverage - name: Archive coverage report - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: code-coverage-report path: ${{ github.workspace }}/build/html @@ -244,7 +244,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On - name: Build @@ -262,7 +262,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -279,7 +279,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10 run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV" if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }} @@ -299,7 +299,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -319,7 +319,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build with libc++ @@ -346,7 +346,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -366,7 +366,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 with: cmakeVersion: 4.3.4 # Clang: the std library module is provided by libc++ (the image's libstdc++ @@ -392,7 +392,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -409,7 +409,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -423,7 +423,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -443,7 +443,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja - name: Build @@ -456,7 +456,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid] steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -465,7 +465,7 @@ jobs: persist-credentials: false # the git-revision-date-localized plugin needs the history; a shallow clone makes the strict build fail fetch-depth: ${{ matrix.target == 'ci_test_build_documentation' && '0' || '1' }} - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 if: matrix.target == 'ci_test_documentation_mermaid' with: node-version: 24 @@ -484,7 +484,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Check that the job runs natively on arm64 run: uname -m && test "$(uname -m)" = aarch64 - name: Run CMake diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 0ac496124..bbc1ef48b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -90,7 +90,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Set extra CXX_FLAGS for latest std_version # /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which # specializes std::tuple (newly diagnosed by the VS2026 v145 toolset) @@ -214,7 +214,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 with: cmakeVersion: 4.3.4 # CMAKE_CXX_FLAGS replaces CMake's MSVC defaults (which include /EHsc), and diff --git a/BUILD.bazel b/BUILD.bazel index 08d242eef..8fa0a9a9c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -25,6 +25,7 @@ cc_library( "include/nlohmann/detail/conversions/from_json.hpp", "include/nlohmann/detail/conversions/to_chars.hpp", "include/nlohmann/detail/conversions/to_json.hpp", + "include/nlohmann/detail/conversions/zmij.hpp", "include/nlohmann/detail/exceptions.hpp", "include/nlohmann/detail/hash.hpp", "include/nlohmann/detail/input/binary_reader.hpp", diff --git a/README.md b/README.md index c9fbe4013..a086db3dc 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ There is also a [**docset**](https://github.com/Kapeli/Dash-User-Contributions/t - **Constructors**: [basic_json](https://json.nlohmann.me/api/basic_json/basic_json), [array](https://json.nlohmann.me/api/basic_json/array), [binary](https://json.nlohmann.me/api/basic_json/binary), [object](https://json.nlohmann.me/api/basic_json/object) - **Object inspection**: [type](https://json.nlohmann.me/api/basic_json/type), [operator value_t](https://json.nlohmann.me/api/basic_json/operator_value_t), [type_name](https://json.nlohmann.me/api/basic_json/type_name), [is_primitive](https://json.nlohmann.me/api/basic_json/is_primitive), [is_structured](https://json.nlohmann.me/api/basic_json/is_structured), [is_null](https://json.nlohmann.me/api/basic_json/is_null), [is_boolean](https://json.nlohmann.me/api/basic_json/is_boolean), [is_number](https://json.nlohmann.me/api/basic_json/is_number), [is_number_integer](https://json.nlohmann.me/api/basic_json/is_number_integer), [is_number_unsigned](https://json.nlohmann.me/api/basic_json/is_number_unsigned), [is_number_float](https://json.nlohmann.me/api/basic_json/is_number_float), [is_object](https://json.nlohmann.me/api/basic_json/is_object), [is_array](https://json.nlohmann.me/api/basic_json/is_array), [is_string](https://json.nlohmann.me/api/basic_json/is_string), [is_binary](https://json.nlohmann.me/api/basic_json/is_binary), [is_discarded](https://json.nlohmann.me/api/basic_json/is_discarded) - **Value access**: [get](https://json.nlohmann.me/api/basic_json/get), [get_to](https://json.nlohmann.me/api/basic_json/get_to), [get_ptr](https://json.nlohmann.me/api/basic_json/get_ptr), [get_ref](https://json.nlohmann.me/api/basic_json/get_ref), [operator ValueType](https://json.nlohmann.me/api/basic_json/operator_ValueType), [get_binary](https://json.nlohmann.me/api/basic_json/get_binary) -- **Element access**: [at](https://json.nlohmann.me/api/basic_json/at), [operator[]](https://json.nlohmann.me/api/basic_json/operator[]), [value](https://json.nlohmann.me/api/basic_json/value), [front](https://json.nlohmann.me/api/basic_json/front), [back](https://json.nlohmann.me/api/basic_json/back) +- **Element access**: [at](https://json.nlohmann.me/api/basic_json/at), [operator[]](https://json.nlohmann.me/api/basic_json/operator%5B%5D/), [value](https://json.nlohmann.me/api/basic_json/value), [front](https://json.nlohmann.me/api/basic_json/front), [back](https://json.nlohmann.me/api/basic_json/back) - **Lookup**: [find](https://json.nlohmann.me/api/basic_json/find), [count](https://json.nlohmann.me/api/basic_json/count), [contains](https://json.nlohmann.me/api/basic_json/contains) - **Iterators**: [begin](https://json.nlohmann.me/api/basic_json/begin), [cbegin](https://json.nlohmann.me/api/basic_json/cbegin), [end](https://json.nlohmann.me/api/basic_json/end), [cend](https://json.nlohmann.me/api/basic_json/cend), [rbegin](https://json.nlohmann.me/api/basic_json/rbegin), [rend](https://json.nlohmann.me/api/basic_json/rend), [crbegin](https://json.nlohmann.me/api/basic_json/crbegin), [crend](https://json.nlohmann.me/api/basic_json/crend), [items](https://json.nlohmann.me/api/basic_json/items) - **Capacity**: [empty](https://json.nlohmann.me/api/basic_json/empty), [size](https://json.nlohmann.me/api/basic_json/size), [max_size](https://json.nlohmann.me/api/basic_json/max_size) @@ -125,7 +125,7 @@ There is also a [**docset**](https://github.com/Kapeli/Dash-User-Contributions/t - **Serialization / Dumping**: [dump](https://json.nlohmann.me/api/basic_json/dump) - **Deserialization / Parsing**: [parse](https://json.nlohmann.me/api/basic_json/parse), [accept](https://json.nlohmann.me/api/basic_json/accept), [sax_parse](https://json.nlohmann.me/api/basic_json/sax_parse) - **JSON Pointer functions**: [flatten](https://json.nlohmann.me/api/basic_json/flatten), [unflatten](https://json.nlohmann.me/api/basic_json/unflatten) -- **JSON Patch functions**: [patch](https://json.nlohmann.me/api/basic_json/patch), [patch_inplace](https://json.nlohmann.me/api/basic_json/patch_inplace), [diff](https://json.nlohmann.me/api/basic_json/diff), [merge_patch](https://json.nlohmann.me/api/basic_json/merge_patch) +- **JSON Patch functions**: [patch](https://json.nlohmann.me/api/basic_json/patch), [patch_inplace](https://json.nlohmann.me/api/basic_json/patch_inplace), [diff](https://json.nlohmann.me/api/basic_json/diff), [merge_patch](https://json.nlohmann.me/api/basic_json/merge_patch), [merge_diff](https://json.nlohmann.me/api/basic_json/merge_diff) - **Static functions**: [meta](https://json.nlohmann.me/api/basic_json/meta), [get_allocator](https://json.nlohmann.me/api/basic_json/get_allocator) - **Binary formats**: [from_bjdata](https://json.nlohmann.me/api/basic_json/from_bjdata), [from_bon8](https://json.nlohmann.me/api/basic_json/from_bon8), [from_bson](https://json.nlohmann.me/api/basic_json/from_bson), [from_cbor](https://json.nlohmann.me/api/basic_json/from_cbor), [from_msgpack](https://json.nlohmann.me/api/basic_json/from_msgpack), [from_ubjson](https://json.nlohmann.me/api/basic_json/from_ubjson), [to_bjdata](https://json.nlohmann.me/api/basic_json/to_bjdata), [to_bon8](https://json.nlohmann.me/api/basic_json/to_bon8), [to_bson](https://json.nlohmann.me/api/basic_json/to_bson), [to_cbor](https://json.nlohmann.me/api/basic_json/to_cbor), [to_msgpack](https://json.nlohmann.me/api/basic_json/to_msgpack), [to_ubjson](https://json.nlohmann.me/api/basic_json/to_ubjson) - **Non-member functions**: [operator<<](https://json.nlohmann.me/api/operator_ltlt/), [operator>>](https://json.nlohmann.me/api/operator_gtgt/), [to_string](https://json.nlohmann.me/api/basic_json/to_string) @@ -360,7 +360,7 @@ std::cout << cpp_string << " == " << cpp_string2 << " == " << j_string.get()`](https://json.nlohmann.me/api/basic_json/get/) to retrieve the originally stored string value. Note the library only supports UTF-8. When you store strings with different encodings in the library, calling [`dump()`](https://json.nlohmann.me/api/basic_json/dump/) may throw an exception unless `json::error_handler_t::replace`, `json::error_handler_t::ignore`, or `json::error_handler_t::keep` are used as error handlers. @@ -687,15 +687,15 @@ json j_result = j_original.patch(j_patch); // calculate a JSON patch from two JSON values json::diff(j_result, j_original); // [ -// { "op":" replace", "path": "/baz", "value": ["one", "two", "three"] }, -// { "op": "remove","path": "/hello" }, +// { "op": "replace", "path": "/baz", "value": ["one", "two", "three"] }, +// { "op": "remove", "path": "/hello" }, // { "op": "add", "path": "/foo", "value": "bar" } // ] ``` ### JSON Merge Patch -The library supports **JSON Merge Patch** ([RFC 7386](https://tools.ietf.org/html/rfc7386)) as a patch format. Instead of using JSON Pointer (see above) to specify values to be manipulated, it describes the changes using a syntax that closely mimics the document being modified. +The library supports **JSON Merge Patch** ([RFC 7396](https://tools.ietf.org/html/rfc7396)) as a patch format. Instead of using JSON Pointer (see above) to specify values to be manipulated, it describes the changes using a syntax that closely mimics the document being modified. ```cpp // a JSON value @@ -725,6 +725,20 @@ j_document.merge_patch(j_patch); // } ``` +Conversely, [`merge_diff`](https://json.nlohmann.me/api/basic_json/merge_diff/) creates a merge patch from two JSON values: + +```cpp +// create a merge patch that turns j_original into j_document +json j_original = R"({"a": "b", "c": {"d": "e", "f": "g"}})"_json; +json::merge_diff(j_original, j_document); +// { +// "a": "z", +// "c": { +// "f": null +// } +// } +``` + ### Implicit conversions Supported types can be implicitly converted to JSON values. @@ -1266,13 +1280,9 @@ The package configuration file, `nlohmann_jsonConfig.cmake`, can be used either To embed the library directly into an existing CMake project, place the entire source tree in a subdirectory and call `add_subdirectory()` in your `CMakeLists.txt` file: ```cmake -# Typically you don't care so much for a third party library's tests to be -# run from your own project's code. -set(JSON_BuildTests OFF CACHE INTERNAL "") - # If you only include this third party in PRIVATE source files, you do not # need to install it when your main project gets installed. -# set(JSON_Install OFF CACHE INTERNAL "") +set(JSON_Install OFF CACHE INTERNAL "") # Don't use include(nlohmann_json/CMakeLists.txt) since that carries with it # unintended consequences that will break the build. It's generally @@ -1287,8 +1297,8 @@ target_link_libraries(foo PRIVATE nlohmann_json::nlohmann_json) ##### Embedded (FetchContent) -Since CMake v3.11, -[FetchContent](https://cmake.org/cmake/help/v3.11/module/FetchContent.html) can +Since CMake v3.14, +[FetchContent](https://cmake.org/cmake/help/v3.14/module/FetchContent.html) can be used to automatically download a release as a dependency at configure time. Example: @@ -1353,7 +1363,9 @@ Use your favorite [**package manager**](https://json.nlohmann.me/integration/pac -  [**Swift Package Manager**](https://json.nlohmann.me/integration/package_managers/#swift-package-manager) `nlohmann/json` -  [**Nuget**](https://json.nlohmann.me/integration/package_managers/#nuget) `nlohmann.json` -  [**Conda**](https://json.nlohmann.me/integration/package_managers/#conda) `nlohmann_json` +- [**MSYS2**](https://json.nlohmann.me/integration/package_managers/#msys2) `mingw-w64-nlohmann-json` -  [**MacPorts**](https://json.nlohmann.me/integration/package_managers/#macports) `nlohmann-json` +- [**build2**](https://json.nlohmann.me/integration/package_managers/#build2) `nlohmann-json` -  [**cpm.cmake**](https://json.nlohmann.me/integration/package_managers/#cpmcmake) `gh:nlohmann/json` -  [**xmake**](https://json.nlohmann.me/integration/package_managers/#xmake) `nlohmann_json` @@ -1385,9 +1397,10 @@ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR I - The class contains the UTF-8 Decoder from Bjoern Hoehrmann which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2008-2009 [Björn Hoehrmann](https://bjoern.hoehrmann.de/) - The class contains a slightly modified version of the Grisu2 algorithm from Florian Loitsch which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2009 [Florian Loitsch](https://florian.loitsch.com/) +- The class contains a port of the shortest double-to-decimal conversion of [Żmij](https://github.com/vitaut/zmij) by Victor Zverovich, including the conversion of the digits to text by Xiang JunBo and the SIMD instruction sequence of Dougall Johnson, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2025 [Victor Zverovich](https://github.com/vitaut) - The class contains a copy of [Hedley](https://nemequ.github.io/hedley/) from Evan Nemerson which is licensed as [CC0-1.0](https://creativecommons.org/publicdomain/zero/1.0/). - The class contains parts of [Google Abseil](https://github.com/abseil/abseil-cpp) which is licensed under the [Apache 2.0 License](https://opensource.org/licenses/Apache-2.0). -- The class contains an adapted version of the Eisel-Lemire algorithm and its table of powers of five from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors +- The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, and its digit comparison for long numbers from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors REUSE Software diff --git a/cmake/ci.cmake b/cmake/ci.cmake index 5affec6c5..d85214922 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -485,6 +485,13 @@ add_custom_target(ci_clang_analyze # Check code with Cppcheck. ############################################################################### +# default_defines.hpp expands the Hedley macros to the compiler's feature-test operators, which Cppcheck does not know. +# Since Cppcheck 2.20, it rejects their invocations in #if conditions instead of evaluating them as 0. +file(WRITE ${CMAKE_BINARY_DIR}/cppcheck_feature_tests.hpp "") +foreach(OPERATOR __has_attribute __has_builtin __has_cpp_attribute __has_declspec_attribute __has_extension __has_feature __has_include __has_warning) + file(APPEND ${CMAKE_BINARY_DIR}/cppcheck_feature_tests.hpp "#define ${OPERATOR}(x) 0\n") +endforeach() + add_custom_target(ci_cppcheck COMMAND mkdir -p cppcheck COMMAND clang -dM -E -x c++ -std=c++11 ${CMAKE_SOURCE_DIR}/include/nlohmann/thirdparty/hedley/hedley.hpp > default_defines.hpp 2> /dev/null @@ -493,7 +500,7 @@ add_custom_target(ci_cppcheck COMMAND venv_cppcheck/bin/cppcheck --enable=warning --check-level=exhaustive --inline-suppr --inconclusive --force --std=c++11 ${PROJECT_SOURCE_DIR}/include/nlohmann/json.hpp -I ${CMAKE_SOURCE_DIR}/include --error-exitcode=1 --relative-paths=${PROJECT_SOURCE_DIR} -j ${N} --include=default_defines.hpp - --cppcheck-build-dir=cppcheck + --include=${CMAKE_BINARY_DIR}/cppcheck_feature_tests.hpp --cppcheck-build-dir=cppcheck -UJSON_CATCH_USER -UJSON_TRY_USER -UJSON_ASSERT -UJSON_INTERNAL_CATCH -UJSON_THROW -DJSON_HAS_CPP_11 -UJSON_HAS_CPP_14 -UJSON_HAS_CPP_17 -UJSON_HAS_CPP_20 -UJSON_HAS_THREE_WAY_COMPARISON COMMENT "Check code with Cppcheck" @@ -610,9 +617,8 @@ foreach(SRC_FILE ${SRC_FILES}) add_executable(single_${RELATIVE_SRC_FILE} EXCLUDE_FROM_ALL ${PROJECT_BINARY_DIR}/src_single/${RELATIVE_SRC_FILE}.cpp) target_include_directories(single_${RELATIVE_SRC_FILE} PRIVATE ${PROJECT_SOURCE_DIR}/include) target_compile_features(single_${RELATIVE_SRC_FILE} PRIVATE cxx_std_11) - if(RELATIVE_SRC_FILE STREQUAL "json" OR RELATIVE_SRC_FILE STREQUAL "json_literals") - # see below: report the diagnostics of json.hpp and json_literals.hpp without --error, so they - # do not fail the build + if(RELATIVE_SRC_FILE STREQUAL "json") + # see below: report the diagnostics of json.hpp without --error, so they do not fail the build set_property(TARGET single_${RELATIVE_SRC_FILE} PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${IWYU_TOOL} -Xiwyu --max_line_length=300) else() set_property(TARGET single_${RELATIVE_SRC_FILE} PROPERTY CXX_INCLUDE_WHAT_YOU_USE "${iwyu_path_and_options}") @@ -626,18 +632,31 @@ foreach(SRC_FILE ${SRC_FILES}) # reporting its diagnostics (informational, via CXX_INCLUDE_WHAT_YOU_USE above) but exclude it # from the hard gate below so a fresh IWYU/compiler combination does not fail this target on a # nondeterministic suggestion for a header that already re-exports everything on purpose. - # json_literals.hpp and json.hpp include each other on purpose (json.hpp includes it at its end - # unless JSON_NO_AUTOMATIC_UDLS is defined), and IWYU, not following the cycle, suggests replacing - # json.hpp with json_fwd.hpp although the literals need the complete basic_json; exclude it, too. - if(NOT RELATIVE_SRC_FILE STREQUAL "json" AND NOT RELATIVE_SRC_FILE STREQUAL "json_literals") + if(NOT RELATIVE_SRC_FILE STREQUAL "json") list(APPEND single_binaries_tus src_single/${RELATIVE_SRC_FILE}.cpp) endif() endforeach() +# The checks above look at each header on its own. Client code that includes only +# must not be told by include analyzers to include another nlohmann header for what it uses +# (issue #5812): json.hpp marks those includes "IWYU pragma: export", and both IWYU and clang-tidy's +# misc-include-cleaner (disabled in .clang-tidy for the library itself) check these clients. +file(GLOB CONSUMER_FILES ${PROJECT_SOURCE_DIR}/tests/iwyu_consumer/*.cpp) +foreach(CONSUMER_FILE ${CONSUMER_FILES}) + get_filename_component(CONSUMER_NAME "${CONSUMER_FILE}" NAME_WE) + add_executable(iwyu_consumer_${CONSUMER_NAME} EXCLUDE_FROM_ALL ${CONSUMER_FILE}) + target_include_directories(iwyu_consumer_${CONSUMER_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/include) + target_compile_features(iwyu_consumer_${CONSUMER_NAME} PRIVATE cxx_std_11) + list(APPEND single_binaries iwyu_consumer_${CONSUMER_NAME}) + list(APPEND single_binaries_tus ${CONSUMER_FILE}) +endforeach() + if(IWYU_TOOL_PY) add_custom_target(ci_single_binaries DEPENDS ${single_binaries} COMMAND ${IWYU_TOOL_PY} -p ${PROJECT_BINARY_DIR} ${single_binaries_tus} -- ${iwyu_options} + COMMAND ${CLANG_TIDY_TOOL} -p ${PROJECT_BINARY_DIR} --checks=-*,misc-include-cleaner --warnings-as-errors=* ${CONSUMER_FILES} + VERBATIM COMMENT "Check if headers are self-contained" ) else() diff --git a/cmake/requirements/requirements-cppcheck.txt b/cmake/requirements/requirements-cppcheck.txt index aa1591128..ecfb21ac7 100644 --- a/cmake/requirements/requirements-cppcheck.txt +++ b/cmake/requirements/requirements-cppcheck.txt @@ -1 +1 @@ -cppcheck==1.5.1 +cppcheck==1.5.2 diff --git a/docs/mkdocs/docs/api/basic_json/array_t.md b/docs/mkdocs/docs/api/basic_json/array_t.md index caa0e9162..f621f7b58 100644 --- a/docs/mkdocs/docs/api/basic_json/array_t.md +++ b/docs/mkdocs/docs/api/basic_json/array_t.md @@ -27,7 +27,7 @@ To store objects in C++, a type is defined by the template parameters explained ### Default type -With the default values for `ArrayType` (`std::vector`) and `AllocatorType` (`std::allocator`), the default value for +With the default values for `ArrayType` (`std::vector`) and `AllocatorType` (`std::allocator`), the default type for `array_t` is: ```cpp diff --git a/docs/mkdocs/docs/api/basic_json/basic_json.md b/docs/mkdocs/docs/api/basic_json/basic_json.md index 78fa3560c..e553509ea 100644 --- a/docs/mkdocs/docs/api/basic_json/basic_json.md +++ b/docs/mkdocs/docs/api/basic_json/basic_json.md @@ -68,6 +68,14 @@ basic_json(basic_json&& other) noexcept; `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with a `key_type` compatible to `string_t` and a `value_type` from which a `basic_json` value can be constructed. - **strings**: `string_t`, string literals, and all compatible string containers can be used. + + !!! note "Rvalue containers are moved from" + + If `val` is an rvalue of a compatible array-like or object-like container other than `array_t`/`object_t` + (e.g., `json j = std::move(vec);` or `json j(std::vector{...});`), the elements are moved into + the JSON value instead of copied. Afterwards, the elements of the source container are in a valid but + unspecified state (**moved-from**). Keys of associative containers are `const` and therefore still copied. + Lvalues and `const` rvalues are copied, and the source is left unchanged. - **numbers**: [`number_integer_t`](number_integer_t.md), [`number_unsigned_t`](number_unsigned_t.md), [`number_float_t`](number_float_t.md), and all convertible number types such as `int`, `size_t`, `int64_t`, `float` or `double` can be used. @@ -236,7 +244,9 @@ basic_json(basic_json&& other) noexcept; 2. No-throw guarantee: this constructor never throws exceptions. 3. Depends on the called constructor. For types directly supported by the library (i.e., all types for which no `to_json()` function was provided), a strong guarantee holds: if an exception is thrown, there are no changes to any - JSON value. + JSON value. **Exception:** when `val` is an rvalue array-like or object-like container, its elements are moved one + by one; if an exception is thrown in the middle of the conversion (e.g., an allocation failure), the JSON value is + unchanged, but the source container may be left partially moved-from. 4. Depends on the called constructor. For types directly supported by the library (i.e., all types for which no `to_json()` function was provided), a strong guarantee holds: if an exception is thrown, there are no changes to any JSON value. diff --git a/docs/mkdocs/docs/api/basic_json/boolean_t.md b/docs/mkdocs/docs/api/basic_json/boolean_t.md index f79a7622e..2fd1beb4e 100644 --- a/docs/mkdocs/docs/api/basic_json/boolean_t.md +++ b/docs/mkdocs/docs/api/basic_json/boolean_t.md @@ -23,7 +23,7 @@ To store boolean values in C++, a type is defined by the template parameter `Bo ### Default type -With the default values for `BooleanType` (`#!cpp bool`), the default value for `boolean_t` is `#!cpp bool`. +With the default values for `BooleanType` (`#!cpp bool`), the default type for `boolean_t` is `#!cpp bool`. ### Storage diff --git a/docs/mkdocs/docs/api/basic_json/diff.md b/docs/mkdocs/docs/api/basic_json/diff.md index ae22454dd..c4e710a7d 100644 --- a/docs/mkdocs/docs/api/basic_json/diff.md +++ b/docs/mkdocs/docs/api/basic_json/diff.md @@ -59,6 +59,7 @@ Currently, only `remove`, `add`, and `replace` operations are generated. - [patch](patch.md) applies a JSON Patch - [patch_inplace](patch_inplace.md) applies a JSON Patch in place - [merge_patch](merge_patch.md) applies a JSON Merge Patch +- [merge_diff](merge_diff.md) creates a diff as a JSON Merge Patch ## Version history diff --git a/docs/mkdocs/docs/api/basic_json/dump.md b/docs/mkdocs/docs/api/basic_json/dump.md index d805f3db1..eb13717e4 100644 --- a/docs/mkdocs/docs/api/basic_json/dump.md +++ b/docs/mkdocs/docs/api/basic_json/dump.md @@ -62,6 +62,9 @@ Linear. ## Notes +Floating-point numbers are written with the fewest digits that read back as the same value (for `#!cpp double`; see +[number handling](../../features/types/number_handling.md#number-serialization)). + Binary values are serialized as an object containing two keys: - "bytes": an array of bytes as integers @@ -97,3 +100,5 @@ Binary values are serialized as an object containing two keys: - Error handlers added in version 3.4.0. - Serialization of binary values added in version 3.8.0. - Error handler `keep` added in version 3.13.0. +- Doubles are written with the shortest digits (Żmij instead of Grisu2) since version 3.13.0; about 0.1% of doubles are + written differently, most of them with fewer digits. diff --git a/docs/mkdocs/docs/api/basic_json/index.md b/docs/mkdocs/docs/api/basic_json/index.md index fdb57105c..8d945de9b 100644 --- a/docs/mkdocs/docs/api/basic_json/index.md +++ b/docs/mkdocs/docs/api/basic_json/index.md @@ -85,7 +85,7 @@ The class satisfies the following concept requirements: - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable): JSON values can be compared with `==`, see [`operator==`](operator_eq.md). - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable): JSON values can be compared with - `<`, see [`operator<`](operator_le.md). + `<`, see [`operator<`](operator_lt.md). - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable): Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of other compatible types, using unqualified function `swap`. - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer): JSON values can be compared against @@ -285,6 +285,7 @@ Access to the JSON value ### JSON Merge Patch functions - [**merge_patch**](merge_patch.md) - applies a JSON Merge Patch +- [**merge_diff**](merge_diff.md) (_static_) - creates a diff as a JSON Merge Patch ## Static functions diff --git a/docs/mkdocs/docs/api/basic_json/is_number_unsigned.md b/docs/mkdocs/docs/api/basic_json/is_number_unsigned.md index 50083164e..4dfb99049 100644 --- a/docs/mkdocs/docs/api/basic_json/is_number_unsigned.md +++ b/docs/mkdocs/docs/api/basic_json/is_number_unsigned.md @@ -6,6 +6,9 @@ constexpr bool is_number_unsigned() const noexcept; This function returns `#!cpp true` if and only if the JSON value is an unsigned integer number. This excludes floating-point and signed integer values. + +If [`number_unsigned_t`](number_unsigned_t.md) is the same type as [`number_integer_t`](number_integer_t.md), every +integer is stored as `number_integer_t`, and this function always returns `#!cpp false`. ## Return value diff --git a/docs/mkdocs/docs/api/basic_json/merge_diff.md b/docs/mkdocs/docs/api/basic_json/merge_diff.md new file mode 100644 index 000000000..b02ac3e54 --- /dev/null +++ b/docs/mkdocs/docs/api/basic_json/merge_diff.md @@ -0,0 +1,79 @@ +# nlohmann::basic_json::merge_diff + +```cpp +static basic_json merge_diff(const basic_json& source, + const basic_json& target); +``` + +Creates a [JSON Merge Patch](https://tools.ietf.org/html/rfc7396) so that value `source` can be changed into the value +`target` by calling the [`merge_patch`](merge_patch.md) function. The patch contains only what differs: changed and +added members with their new values, and removed members with the value `#!json null`. Nested objects are compared +member by member; any other value that differs, including an array, is replaced as a whole. + +For two JSON values `source` and `target`, where `target` contains no object member whose value is `#!json null`, the +following code always yields `#!cpp true` (for [`ordered_json`](../ordered_json.md), see the notes below): +```cpp +basic_json patched = source; +patched.merge_patch(merge_diff(source, target)); +patched == target; +``` + +## Parameters + +`source` (in) +: JSON value to compare from + +`target` (in) +: JSON value to compare against + +## Return value + +a JSON Merge Patch to convert the `source` to `target` + +## Exception safety + +Strong guarantee: `source` and `target` are never modified. + +## Complexity + +Linear in the total number of values (including nested ones) in `source` and `target`, times the cost of a key-based +object operation (looking up a key, or adding one to the patch): logarithmic for [`json`](../json.md), and linear for +[`ordered_json`](../ordered_json.md) (see [`ordered_map` complexity](../ordered_map.md#complexity)). For +`ordered_json`, diffing two objects with `n` and `m` members therefore takes O((n+m)²). + +## Notes + +- A merge patch uses `#!json null` to remove a member, so it cannot set a member to `#!json null`. As in RFC 7396, a + `#!json null` member of `target` is treated as absent: the patch removes the member from `source` instead of setting + it to `#!json null`. A member that is `#!json null` in both `source` and `target` is unchanged and left out of the + patch, so `source` keeps it. +- A merge patch cannot reorder object members, and [`merge_patch`](merge_patch.md) appends added members at the end. + For [`ordered_json`](../ordered_json.md), the patched value therefore has the same members as `target`, but not + necessarily in the same order, and may compare unequal to it. The patch lists changed and removed members in the + order of `source`, followed by added members in the order of `target`. + +## Examples + +??? example + + The following code shows how a JSON Merge Patch is created as a diff for two JSON values. + + ```cpp + --8<-- "examples/merge_diff.cpp" + ``` + + Output: + + ```json + --8<-- "examples/merge_diff.output" + ``` + +## See also + +- [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396) +- [merge_patch](merge_patch.md) applies a JSON Merge Patch +- [diff](diff.md) creates a diff as a JSON Patch + +## Version history + +- Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/merge_patch.md b/docs/mkdocs/docs/api/basic_json/merge_patch.md index dbda23f42..00e1bc8e5 100644 --- a/docs/mkdocs/docs/api/basic_json/merge_patch.md +++ b/docs/mkdocs/docs/api/basic_json/merge_patch.md @@ -67,6 +67,7 @@ returned by `#!cpp (*this)[key]`); it is read as it was when `merge_patch()` was - [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396) - [patch](patch.md) apply a JSON patch +- [merge_diff](merge_diff.md) creates a diff as a JSON Merge Patch ## Version history diff --git a/docs/mkdocs/docs/api/basic_json/number_float_t.md b/docs/mkdocs/docs/api/basic_json/number_float_t.md index 282bb6123..cf482d6f0 100644 --- a/docs/mkdocs/docs/api/basic_json/number_float_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_float_t.md @@ -23,9 +23,10 @@ type to use. ## Template parameters `NumberFloatType` -: the type to store floating-point numbers. Parsing and serialization are implemented in terms of - `#!cpp std::strtof`/`#!cpp std::strtod`/`#!cpp std::strtold` and `#!cpp std::snprintf`, so the type must be - `#!cpp float`, `#!cpp double`, or `#!cpp long double`. The +: the type to store floating-point numbers. The type must be `#!cpp float`, `#!cpp double`, or + `#!cpp long double`. The parser converts `#!cpp float`, `#!cpp double`, and a `#!cpp long double` that is IEEE 754 + binary64 itself. It converts other `#!cpp long double` formats with `#!cpp std::from_chars` where available, or + with `#!cpp std::strtold` otherwise. Serialization falls back to `#!cpp std::snprintf`. The [binary formats](../../features/binary_formats/index.md) additionally require `#!cpp float` or `#!cpp double`, because they have no encoding for `#!cpp long double`. See [Template Parameter Requirements](../../features/types/template_parameters.md#numberfloattype). @@ -34,7 +35,7 @@ type to use. ### Default type -With the default values for `NumberFloatType` (`double`), the default value for `number_float_t` is `#!cpp double`. +With the default values for `NumberFloatType` (`double`), the default type for `number_float_t` is `#!cpp double`. ### Default behavior @@ -51,9 +52,11 @@ With the default values for `NumberFloatType` (`double`), the default value for > interoperability can be achieved by implementations that expect no more precision or range than these provide, in the > sense that implementations will approximate JSON numbers within the expected precision. -This implementation does exactly follow this approach, as it uses double precision floating-point numbers. Note values -smaller than `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` will be stored as NaN internally -and be serialized to `null`. +This implementation does exactly follow this approach, as it uses double precision floating-point numbers by default. +Note that with the default `double`, numbers smaller than `-1.79769313486232e+308` or greater than +`1.79769313486232e+308` are rejected during parsing with +[`out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406). Non-finite values (NaN and infinity) can +only be stored by assigning them directly and are serialized to `null`. During deserialization (from JSON text or any of the binary formats), a finite number that does not fit into `number_float_t` is rejected with [`out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406), for diff --git a/docs/mkdocs/docs/api/basic_json/number_integer_t.md b/docs/mkdocs/docs/api/basic_json/number_integer_t.md index 48d2873ed..6bd18a74b 100644 --- a/docs/mkdocs/docs/api/basic_json/number_integer_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_integer_t.md @@ -31,7 +31,7 @@ to use. ### Default type -With the default values for `NumberIntegerType` (`std::int64_t`), the default value for `number_integer_t` is +With the default values for `NumberIntegerType` (`std::int64_t`), the default type for `number_integer_t` is `#!cpp std::int64_t`. ### Default behavior diff --git a/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md b/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md index 81cc00f77..cd8044bd3 100644 --- a/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md @@ -25,14 +25,15 @@ the type to use. `NumberUnsignedType` : the type to store unsigned integers. It must be an **unsigned integral** type (`#!cpp std::is_integral`) with a `#!cpp std::numeric_limits` specialization, and it must be able to represent the absolute value of every - [`number_integer_t`](number_integer_t.md) value. See + [`number_integer_t`](number_integer_t.md) value. Alternatively, it can be the same type as + [`number_integer_t`](number_integer_t.md); then every integer is stored as `number_integer_t`. See [Template Parameter Requirements](../../features/types/template_parameters.md#numberintegertype-and-numberunsignedtype). ## Notes ### Default type -With the default values for `NumberUnsignedType` (`std::uint64_t`), the default value for `number_unsigned_t` is +With the default values for `NumberUnsignedType` (`std::uint64_t`), the default type for `number_unsigned_t` is `#!cpp std::uint64_t`. ### Default behavior @@ -89,3 +90,4 @@ Integer number values are stored directly inside a `basic_json` type. ## Version history - Added in version 2.0.0. +- Can be the same type as `number_integer_t` since version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/object_t.md b/docs/mkdocs/docs/api/basic_json/object_t.md index 204e5ab28..66fa9c110 100644 --- a/docs/mkdocs/docs/api/basic_json/object_t.md +++ b/docs/mkdocs/docs/api/basic_json/object_t.md @@ -36,7 +36,7 @@ To store objects in C++, a type is defined by the template parameters described ### Default type With the default values for `ObjectType` (`std::map`), `StringType` (`std::string`), and `AllocatorType` -(`std::allocator`), the default value for `object_t` is: +(`std::allocator`), the default type for `object_t` is: ```cpp // until C++14 diff --git a/docs/mkdocs/docs/api/basic_json/operator_eq.md b/docs/mkdocs/docs/api/basic_json/operator_eq.md index 58a7e74da..778945f38 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_eq.md +++ b/docs/mkdocs/docs/api/basic_json/operator_eq.md @@ -11,18 +11,17 @@ template bool operator==(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2) // since C++20 -class basic_json { - bool operator==(const_reference rhs) const noexcept; // (1) +bool operator==(const_reference rhs) const noexcept; // (1) - template - bool operator==(ScalarType rhs) const noexcept(/* see below */); // (2) -}; +template +bool operator==(ScalarType rhs) const noexcept(/* see below */); // (2) ``` 1. Compares two JSON values for equality according to the following rules: - Two JSON values are equal if (1) neither value is discarded, and (2) they are of the same type and their stored values are the same according to their respective `operator==`. - - Integer and floating-point numbers are automatically converted before comparison. + - Integer and floating-point numbers are compared by their exact values, without converting the integer to + floating-point. For instance, `9007199254740993` and `9007199254740992.0` are not equal. 2. Compares a JSON value and a scalar or a scalar and a JSON value for equality by converting the scalar to a JSON value and comparing both JSON values according to 1. diff --git a/docs/mkdocs/docs/api/basic_json/operator_lt.md b/docs/mkdocs/docs/api/basic_json/operator_lt.md index 1b8e65225..baa04964c 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_lt.md +++ b/docs/mkdocs/docs/api/basic_json/operator_lt.md @@ -15,7 +15,8 @@ bool operator<(ScalarType lhs, const const_reference rhs) noexcept(/* see below following rules: - If either operand is discarded, the comparison yields `#!cpp false`. - If both operands have the same type, the values are compared using their respective `operator<`. - - Integer and floating-point numbers are automatically converted before comparison. + - Integer and floating-point numbers are compared by their exact values, without converting the integer to + floating-point. For instance, `9007199254740992.0` is less than `9007199254740993`. - In case `lhs` and `rhs` have different types, the values are ignored and the order of the types is considered, which is: 1. null diff --git a/docs/mkdocs/docs/api/basic_json/operator_spaceship.md b/docs/mkdocs/docs/api/basic_json/operator_spaceship.md index 47ca22484..731372f97 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_spaceship.md +++ b/docs/mkdocs/docs/api/basic_json/operator_spaceship.md @@ -2,22 +2,18 @@ ```cpp // since C++20 -class basic_json { - std::partial_ordering operator<=>(const_reference rhs) const noexcept; // (1) +std::partial_ordering operator<=>(const_reference rhs) const noexcept; // (1) - template - std::partial_ordering operator<=>(const ScalarType rhs) const noexcept(/* see below */); // (2) -}; +template +std::partial_ordering operator<=>(const ScalarType rhs) const noexcept(/* see below */); // (2) ``` 1. 3-way compares two JSON values producing a result of type `std::partial_ordering` according to the following rules: - Two JSON values compare with a result of `std::partial_ordering::unordered` if either value is discarded. - If both JSON values are of the same type, the result is produced by 3-way comparing their stored values using their respective `operator<=>`. - - Integer and floating-point numbers are converted to their common type and then 3-way compared using their - respective `operator<=>`. - For instance, comparing an integer and a floating-point value will 3-way compare the first value converted to - floating-point with the second value. + - Integer and floating-point numbers are 3-way compared by their exact values, without converting the integer to + floating-point. For instance, `9007199254740993` compares greater than `9007199254740992.0`. - Otherwise, yields a result by comparing the type (see [`value_t`](value_t.md)). 2. 3-way compares a JSON value and a scalar or a scalar and a JSON value by converting the scalar to a JSON value and diff --git a/docs/mkdocs/docs/api/basic_json/patch.md b/docs/mkdocs/docs/api/basic_json/patch.md index 951ee46e6..ed26135a9 100644 --- a/docs/mkdocs/docs/api/basic_json/patch.md +++ b/docs/mkdocs/docs/api/basic_json/patch.md @@ -45,7 +45,7 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va pointer inside the patch cannot be resolved, e.g., `-` in a "remove" operation or `1a` for an array; example: `"unresolved reference token '-'"`. - Throws [`out_of_range.405`](../../home/exceptions.md#jsonexceptionout_of_range405) if JSON pointer has no parent - ("add", "remove", "move") + ("remove", "move") - Throws [`out_of_range.411`](../../home/exceptions.md#jsonexceptionout_of_range411) if an "add" operation's target location has a parent that is neither an object nor an array. - Throws [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) if a "remove" operation's target diff --git a/docs/mkdocs/docs/api/basic_json/patch_inplace.md b/docs/mkdocs/docs/api/basic_json/patch_inplace.md index dff8b095b..5b0d501bb 100644 --- a/docs/mkdocs/docs/api/basic_json/patch_inplace.md +++ b/docs/mkdocs/docs/api/basic_json/patch_inplace.md @@ -41,7 +41,7 @@ No guarantees, value may be corrupted by an unsuccessful patch operation. pointer inside the patch cannot be resolved, e.g., `-` in a "remove" operation or `1a` for an array; example: `"unresolved reference token '-'"`. - Throws [`out_of_range.405`](../../home/exceptions.md#jsonexceptionout_of_range405) if JSON pointer has no parent - ("add", "remove", "move") + ("remove", "move") - Throws [`out_of_range.411`](../../home/exceptions.md#jsonexceptionout_of_range411) if an "add" operation's target location has a parent that is neither an object nor an array. - Throws [`out_of_range.413`](../../home/exceptions.md#jsonexceptionout_of_range413) if a "remove" operation's target diff --git a/docs/mkdocs/docs/api/basic_json/string_t.md b/docs/mkdocs/docs/api/basic_json/string_t.md index 13177cef8..b65e95508 100644 --- a/docs/mkdocs/docs/api/basic_json/string_t.md +++ b/docs/mkdocs/docs/api/basic_json/string_t.md @@ -32,7 +32,7 @@ JSON class into byte-sized characters during deserialization. ### Default type -With the default values for `StringType` (`std::string`), the default value for `string_t` is `#!cpp std::string`. +With the default values for `StringType` (`std::string`), the default type for `string_t` is `#!cpp std::string`. ### Encoding diff --git a/docs/mkdocs/docs/api/json_pointer/operator_eq.md b/docs/mkdocs/docs/api/json_pointer/operator_eq.md index a9f40891f..44cda4354 100644 --- a/docs/mkdocs/docs/api/json_pointer/operator_eq.md +++ b/docs/mkdocs/docs/api/json_pointer/operator_eq.md @@ -18,13 +18,11 @@ bool operator==( const json_pointer& rhs); // (2) // since C++20 -class json_pointer { - template - bool operator==( - const json_pointer& rhs) const noexcept; // (1) +template +bool operator==( + const json_pointer& rhs) const noexcept; // (1) - bool operator==(const string_t& rhs) const; // (2) -}; +bool operator==(const string_t& rhs) const; // (2) ``` 1. Compares two JSON pointers for equality by comparing their reference tokens. diff --git a/docs/mkdocs/docs/api/json_pointer/operator_spaceship.md b/docs/mkdocs/docs/api/json_pointer/operator_spaceship.md index 2ec917b72..ffc3f0e93 100644 --- a/docs/mkdocs/docs/api/json_pointer/operator_spaceship.md +++ b/docs/mkdocs/docs/api/json_pointer/operator_spaceship.md @@ -2,10 +2,8 @@ ```cpp // since C++20 -class json_pointer { - template - std::strong_ordering operator<=>(const json_pointer& rhs) const noexcept; // *NOPAD* -}; +template +std::strong_ordering operator<=>(const json_pointer& rhs) const noexcept; // *NOPAD* ``` 3-way compares two JSON pointers by lexicographically comparing their sequences of reference tokens: corresponding diff --git a/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md b/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md index e61ce1a3e..381e1c644 100644 --- a/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md +++ b/docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md @@ -18,7 +18,8 @@ When enabled, comparisons involving at least one discarded JSON value yield resu | `>=` | `#!cpp true` | | `>` | `#!cpp false` | -Otherwise, comparisons involving at least one discarded JSON value always yield `#!cpp false`. +Otherwise, comparisons involving at least one discarded JSON value yield `#!cpp false`, except for `!=`, which is the +negation of `==` and yields `#!cpp true`. ## Default definition diff --git a/docs/mkdocs/docs/community/assurance_case.md b/docs/mkdocs/docs/community/assurance_case.md index 46ebd5ebe..54da5c04c 100644 --- a/docs/mkdocs/docs/community/assurance_case.md +++ b/docs/mkdocs/docs/community/assurance_case.md @@ -54,10 +54,9 @@ flowchart LR user code. The destructor does not recurse, so destroying a deeply nested value does not exhaust the stack. - **Bounded recursion.** The JSON parser and the binary readers keep their state in explicit stacks instead of recursing per nesting level. Operations that walk a value, such as [`dump`](../api/basic_json/dump.md), copying, - comparison, hashing, and [`merge_patch`](../api/basic_json/merge_patch.md), recurse only up to a fixed depth and - continue with an explicit stack below it. Some operations, such as [`diff`](../api/basic_json/diff.md), - [`flatten`](../api/basic_json/flatten.md), and the binary writers, still recurse once per nesting level; work on them - is in progress. Applications that process untrusted input can limit its nesting depth with a + comparison, hashing, [`merge_patch`](../api/basic_json/merge_patch.md), [`diff`](../api/basic_json/diff.md), + [`flatten`](../api/basic_json/flatten.md), and the binary writers, recurse only up to a fixed depth and continue with + an explicit stack below it. Applications that process untrusted input can limit its nesting depth with a [parser callback](../features/parsing/parser_callbacks.md). - **Invariants are checked.** The class invariant (for instance, that the pointer for the stored type is never null) is checked with runtime assertions throughout the test suite. diff --git a/docs/mkdocs/docs/examples/merge_diff.cpp b/docs/mkdocs/docs/examples/merge_diff.cpp new file mode 100644 index 000000000..22ae9b087 --- /dev/null +++ b/docs/mkdocs/docs/examples/merge_diff.cpp @@ -0,0 +1,51 @@ +#include +#include +#include + +using json = nlohmann::json; +using namespace nlohmann::literals; + +int main() +{ + // the source document + json source = R"( + { + "title": "Goodbye!", + "author": { + "givenName": "John", + "familyName": "Doe" + }, + "tags": [ + "example", + "sample" + ], + "content": "This will be unchanged" + } + )"_json; + + // the target document + json target = R"( + { + "title": "Hello!", + "author": { + "givenName": "John" + }, + "tags": [ + "example" + ], + "content": "This will be unchanged", + "phoneNumber": "+01-123-456-7890" + } + )"_json; + + // create the patch + json patch = json::merge_diff(source, target); + + // roundtrip + json patched_source = source; + patched_source.merge_patch(patch); + + // output patch and roundtrip result + std::cout << std::setw(4) << patch << "\n\n" + << std::setw(4) << patched_source << std::endl; +} diff --git a/docs/mkdocs/docs/examples/merge_diff.output b/docs/mkdocs/docs/examples/merge_diff.output new file mode 100644 index 000000000..ca7c59bf2 --- /dev/null +++ b/docs/mkdocs/docs/examples/merge_diff.output @@ -0,0 +1,22 @@ +{ + "author": { + "familyName": null + }, + "phoneNumber": "+01-123-456-7890", + "tags": [ + "example" + ], + "title": "Hello!" +} + +{ + "author": { + "givenName": "John" + }, + "content": "This will be unchanged", + "phoneNumber": "+01-123-456-7890", + "tags": [ + "example" + ], + "title": "Hello!" +} diff --git a/docs/mkdocs/docs/features/binary_values.md b/docs/mkdocs/docs/features/binary_values.md index 98e8e71c3..b4c3bab6d 100644 --- a/docs/mkdocs/docs/features/binary_values.md +++ b/docs/mkdocs/docs/features/binary_values.md @@ -267,9 +267,10 @@ unsigned 8-bit integer. If no subtype is given, the generic binary subtype 0x00 ### CBOR -[CBOR](binary_formats/cbor.md) supports binary values, but no subtypes. Subtypes will be serialized as tags. Any binary -value will be serialized as byte strings. The library will choose the smallest representation using the length of the -byte array. +[CBOR](binary_formats/cbor.md) supports binary values and subtypes. Any binary value will be serialized as a byte +string; a subtype is serialized as a tag preceding it. When parsing, tags are only read back as subtypes with +`cbor_tag_handler_t::store` (see [`from_cbor`](../api/basic_json/from_cbor.md)). The library will choose the smallest +representation using the length of the byte array. ??? example "Example: serialize a binary value to CBOR" diff --git a/docs/mkdocs/docs/features/macros.md b/docs/mkdocs/docs/features/macros.md index 5df3671e2..0f676b501 100644 --- a/docs/mkdocs/docs/features/macros.md +++ b/docs/mkdocs/docs/features/macros.md @@ -243,9 +243,9 @@ See [full documentation of `JSON_USE_GLOBAL_UDLS`](../api/macros/json_use_global ## `JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON` -When defined to `1`, the library restores the legacy behavior in which a discarded value compared equal to itself. This -behavior is [deprecated](../integration/migration_guide.md#miscellaneous-functions) and switched off (`0`) by -default. +When defined to `1`, the library restores the legacy behavior in which comparing a discarded value with `<=` or `>=` +yields `#!cpp true` instead of `#!cpp false`. This behavior is +[deprecated](../integration/migration_guide.md#miscellaneous-functions) and switched off (`0`) by default. See [full documentation of `JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON`](../api/macros/json_use_legacy_discarded_value_comparison.md). diff --git a/docs/mkdocs/docs/features/merge_patch.md b/docs/mkdocs/docs/features/merge_patch.md index c0a316e65..ac732c7bf 100644 --- a/docs/mkdocs/docs/features/merge_patch.md +++ b/docs/mkdocs/docs/features/merge_patch.md @@ -1,6 +1,6 @@ # JSON Merge Patch -The library supports JSON Merge Patch ([RFC 7386](https://tools.ietf.org/html/rfc7386)) as a patch format. +The library supports JSON Merge Patch ([RFC 7396](https://tools.ietf.org/html/rfc7396)) as a patch format. The merge patch format is primarily intended for use with the HTTP PATCH method as a means of describing a set of modifications to a target resource's content. This function applies a merge patch to the current JSON value. @@ -9,6 +9,9 @@ syntax that closely mimics the document being modified. Unlike [JSON Patch](json express every kind of change (e.g., it cannot reorder array elements or remove a specific array element), but it is easier to read and write for object-shaped documents. +The function [`merge_diff`](../api/basic_json/merge_diff.md) creates a merge patch that changes one JSON value into +another. + !!! tip "Not a general deep merge" A merge patch is not a general deep merge: a `#!json null` value in the patch deletes the key from the target. @@ -35,4 +38,5 @@ easier to read and write for object-shaped documents. - [JSON Patch and Diff](json_patch.md) - a more expressive alternative that describes a sequence of operations - [JSON Pointer](json_pointer.md) - the addressing scheme used by JSON Patch - Function [`merge_patch`](../api/basic_json/merge_patch.md) +- Function [`merge_diff`](../api/basic_json/merge_diff.md) - Function [`update`](../api/basic_json/update.md) - merge objects, optionally recursively diff --git a/docs/mkdocs/docs/features/modifying_values.md b/docs/mkdocs/docs/features/modifying_values.md index cf763a4b3..10814ac2e 100644 --- a/docs/mkdocs/docs/features/modifying_values.md +++ b/docs/mkdocs/docs/features/modifying_values.md @@ -66,7 +66,7 @@ config.update(user_settings, true); // {"log":{"file":"app.log","level":"debug"},"retries":3} ``` -[JSON Merge Patch](merge_patch.md) ([RFC 7386](https://tools.ietf.org/html/rfc7386)) also merges objects recursively, +[JSON Merge Patch](merge_patch.md) ([RFC 7396](https://tools.ietf.org/html/rfc7396)) also merges objects recursively, but it is a different tool: a `#!json null` in the patch means "remove this key". It is meant for applying merge patch documents (e.g., received via HTTP PATCH). To merge configuration-like objects, use `#!cpp update(..., true)`. To apply a sequence of well-defined edit operations, see [JSON Patch](json_patch.md). @@ -90,6 +90,6 @@ a.erase(1); // [1,3,4] (erase by index) - [`push_back`](../api/basic_json/push_back.md) / [`emplace_back`](../api/basic_json/emplace_back.md) - append to an array - [`emplace`](../api/basic_json/emplace.md) - insert into an object if the key is absent - [`update`](../api/basic_json/update.md) - merge objects (shallow, or recursive with `merge_objects`) -- [`merge_patch`](../api/basic_json/merge_patch.md) - apply an RFC 7386 merge patch +- [`merge_patch`](../api/basic_json/merge_patch.md) - apply an RFC 7396 merge patch - [`erase`](../api/basic_json/erase.md) / [`clear`](../api/basic_json/clear.md) - remove elements - [JSON Patch and Diff](json_patch.md) and [JSON Merge Patch](merge_patch.md) - structured modifications diff --git a/docs/mkdocs/docs/features/parsing/untrusted_input.md b/docs/mkdocs/docs/features/parsing/untrusted_input.md index d39a4179e..9c5fa4c9e 100644 --- a/docs/mkdocs/docs/features/parsing/untrusted_input.md +++ b/docs/mkdocs/docs/features/parsing/untrusted_input.md @@ -50,19 +50,13 @@ stack, however deeply it is nested. Once a value has been parsed, operations that walk it recursively -- serializing it with [`dump`](../../api/basic_json/dump.md), hashing it, copying it, comparing two values with `#!cpp ==`, `#!cpp <`, or (in -C++20) `#!cpp <=>`, merging with [`update`](../../api/basic_json/update.md), and applying a -[`merge_patch`](../../api/basic_json/merge_patch.md) -- descend at most 128 levels on the call stack and continue +C++20) `#!cpp <=>`, merging with [`update`](../../api/basic_json/update.md), applying a +[`merge_patch`](../../api/basic_json/merge_patch.md), creating a patch with [`diff`](../../api/basic_json/diff.md) or +[`merge_diff`](../../api/basic_json/merge_diff.md), [`flatten`](../../api/basic_json/flatten.md), and the binary +writers (`to_cbor`, `to_msgpack`, ...) -- descend at most 128 levels on the call stack and continue below that with an explicit stack instead, so none of them can exhaust the stack either, however deeply the value is nested. Destroying a value (its destructor) never recurses at all, regardless of nesting depth, for the same reason. -!!! note "Not every operation is bounded yet" - - [`diff`](../../api/basic_json/diff.md), [`flatten`](../../api/basic_json/flatten.md), and the binary writers - (`to_cbor`, `to_msgpack`, ...) still recurse once per nesting level; this is called out as work in progress in the - [assurance case](../../community/assurance_case.md#secure-design). A value deep enough to matter for these - operations would typically first have to survive parsing without hitting a self-imposed depth limit, as described - above. - ## Input size The library does not limit the overall size of a JSON text; a value nested or wide enough will use memory diff --git a/docs/mkdocs/docs/features/serialization.md b/docs/mkdocs/docs/features/serialization.md index 1190c9005..5e1aedac0 100644 --- a/docs/mkdocs/docs/features/serialization.md +++ b/docs/mkdocs/docs/features/serialization.md @@ -52,7 +52,7 @@ with `\uXXXX` sequences, so that the output contains only ASCII characters: ```cpp json j = "苹果"; j.dump(); // "苹果" -j.dump(-1, ' ', true); // "苹果" +j.dump(-1, ' ', true); // "\u82f9\u679c" ``` ## Handling invalid UTF-8 diff --git a/docs/mkdocs/docs/features/types/index.md b/docs/mkdocs/docs/features/types/index.md index 354acd5ac..67e97df6e 100644 --- a/docs/mkdocs/docs/features/types/index.md +++ b/docs/mkdocs/docs/features/types/index.md @@ -120,7 +120,7 @@ requirements are collected in [Template Parameter Requirements](template_paramet ### Default type -With the default values for *ObjectType* (`std::map`), *StringType* (`std::string`), and *AllocatorType* (`std::allocator`), the default value for `object_t` is: +With the default values for *ObjectType* (`std::map`), *StringType* (`std::string`), and *AllocatorType* (`std::allocator`), the default type for `object_t` is: ```cpp std::map< @@ -177,7 +177,7 @@ json j = m; ### Default type -With the default values for *ArrayType* (`std::vector`) and *AllocatorType* (`std::allocator`), the default value for `array_t` is: +With the default values for *ArrayType* (`std::vector`) and *AllocatorType* (`std::allocator`), the default type for `array_t` is: ```cpp std::vector< @@ -209,7 +209,7 @@ Unicode values are split by the JSON class into byte-sized characters during des ### Default type -With the default values for *StringType* (`std::string`), the default value for `string_t` is `#!cpp std::string`. +With the default values for *StringType* (`std::string`), the default type for `string_t` is `#!cpp std::string`. ### Encoding @@ -234,7 +234,7 @@ String values are stored as pointers in a `basic_json` type. That is, for any ac ### Default type -With the default values for *BooleanType* (`#!cpp bool`), the default value for `boolean_t` is `#!cpp bool`. +With the default values for *BooleanType* (`#!cpp bool`), the default type for `boolean_t` is `#!cpp bool`. ### Storage @@ -252,9 +252,9 @@ This description includes both integer and floating-point numbers. However, C++ ### Default types -With the default values for *NumberIntegerType* (`std::int64_t`), the default value for `number_integer_t` is `std::int64_t`. -With the default values for *NumberUnsignedType* (`std::uint64_t`), the default value for `number_unsigned_t` is `std::uint64_t`. -With the default values for *NumberFloatType* (`#!cpp double`), the default value for `number_float_t` is `#!cpp double`. +With the default values for *NumberIntegerType* (`std::int64_t`), the default type for `number_integer_t` is `std::int64_t`. +With the default values for *NumberUnsignedType* (`std::uint64_t`), the default type for `number_unsigned_t` is `std::uint64_t`. +With the default values for *NumberFloatType* (`#!cpp double`), the default type for `number_float_t` is `#!cpp double`. ### Default behavior @@ -281,7 +281,7 @@ As this range is a subrange of the exactly supported range [`INT64_MIN`, `INT64_ > This specification allows implementations to set limits on the range and precision of numbers accepted. Since software that implements IEEE 754-2008 binary64 (double precision) numbers is generally available and widely used, good interoperability can be achieved by implementations that expect no more precision or range than these provide, in the sense that implementations will approximate JSON numbers within the expected precision. -This implementation does exactly follow this approach, as it uses double precision floating-point numbers. Note values smaller than `#!c -1.79769313486232e+308` and values greater than `#!c 1.79769313486232e+308` will be stored as NaN internally and be serialized to `#!json null`. +This implementation does exactly follow this approach, as it uses double precision floating-point numbers by default (see *NumberFloatType* above). Note that with the default `#!cpp double`, numbers smaller than `#!c -1.79769313486232e+308` or greater than `#!c 1.79769313486232e+308` are rejected during parsing with [`out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406). Non-finite values (NaN and infinity) can only be stored by assigning them directly and are serialized to `#!json null`. ### Storage diff --git a/docs/mkdocs/docs/features/types/number_handling.md b/docs/mkdocs/docs/features/types/number_handling.md index 13b6dfd43..d11ad2747 100644 --- a/docs/mkdocs/docs/features/types/number_handling.md +++ b/docs/mkdocs/docs/features/types/number_handling.md @@ -82,12 +82,13 @@ flowchart TD - Numbers with a decimal digit or scientific notation are always stored as `#!c double`. - The number types can be changed, see [Template number types](#template-number-types). - - Integers are converted by the library's own digit parser. Floating-point numbers are converted with - [`std::from_chars`](https://en.cppreference.com/w/cpp/utility/from_chars) if the library is compiled with C++17 - and the standard library supports it, then with an exact fast path for `#!c double` values with few significant - digits, and otherwise with the locale-aware - [`std::strtod`](https://en.cppreference.com/w/cpp/string/byte/strtof) (`std::strtof`/`std::strtold` for the - other floating-point types). Before version 3.13.0, the conversion was realized by + - The library converts integers and floating-point numbers itself, independent of the locale. Floating-point + numbers are correctly rounded (to nearest, ties to even). Only a `#!c long double` that is not IEEE 754 binary64 + (e.g., the 80-bit x87 format) is converted with `#!cpp std::from_chars` where available, or else with + [`std::strtold`](https://en.cppreference.com/w/cpp/string/byte/strtof). For that call, the library temporarily + replaces the `.` with the decimal point of the current locale (which may be longer than one byte, e.g., in + `fa_IR.UTF-8`), so the result does not depend on the locale either. Changing the locale in another thread during + parsing is undefined behavior of the C library, though. Before version 3.13.0, the conversion was realized by [`std::strtoull`](https://en.cppreference.com/w/cpp/string/byte/strtoul), [`std::strtoll`](https://en.cppreference.com/w/cpp/string/byte/strtol), and `std::strtod`, respectively. @@ -100,10 +101,10 @@ flowchart TD ### Number limits - Any 64-bit signed or unsigned integer can be stored without loss of precision. -- Numbers exceeding the limits of `#!c double` (i.e., numbers that after conversion via -[`std::strtod`](https://en.cppreference.com/w/cpp/string/byte/strtof) are not satisfying +- Numbers exceeding the limits of `#!c double` (i.e., numbers whose rounded value is not satisfying [`std::isfinite`](https://en.cppreference.com/w/cpp/numeric/math/isfinite) such as `#!c 1E400`) will throw exception -[`json.exception.out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406) during parsing. +[`json.exception.out_of_range.406`](../../home/exceptions.md#jsonexceptionout_of_range406) during parsing. Numbers too +small for `#!c double` (such as `#!c 1E-400`) become zero, with the sign of the number. - Floating-point numbers are rounded to the next number representable as `double`. For instance `#!c 3.141592653589793238462643383279` is stored as [`0x400921fb54442d18`](https://float.exposed/0x400921fb54442d18). This is the same behavior as the code `#!c double x = 3.141592653589793238462643383279;`. @@ -133,9 +134,10 @@ That is, `-0` is stored as a signed integer, but the serialization does not repr ### Number serialization - Integer numbers are serialized as is; that is, no scientific notation is used. -- Floating-point numbers are serialized as specified by the `#!c %g` printf modifier with - [`std::numeric_limits::max_digits10`](https://en.cppreference.com/w/cpp/types/numeric_limits/max_digits10) - significant digits. The rationale is to use the shortest representation while still allowing round-tripping. +- Floating-point numbers are serialized with the fewest digits that read back as the same value (the closest such + digits if there are several), in the layout of the `#!c %g` printf modifier: `#!c 1.5`, `#!c 100.0`, `#!c 1e+100`. + Doubles are converted with the algorithm of [Żmij](https://github.com/vitaut/zmij), floats with Grisu2, which + can write more digits than necessary. !!! hint "Notes regarding precision of floating-point numbers" @@ -343,8 +345,11 @@ The number types can be changed with template parameters. in case of overflow. - The type for unsigned integers must be convertible from `#!c unsigned long long`. The type for floating-point numbers is used in case of overflow. - - The types for signed and unsigned integers must be distinct, see - [#2573](https://github.com/nlohmann/json/issues/2573). + - The type for unsigned integers can also be the same signed type as the type for signed integers (e.g., + `#!c std::int64_t` for both). Then every integer is stored as a signed integer, and + [`is_number_unsigned()`](../../api/basic_json/is_number_unsigned.md) is always `#!c false`, see + [Template Parameter Requirements](template_parameters.md#same-type-for-both). + - The type for floating-point numbers must be distinct from the integer types. - Only `#!c double`, `#!c float`, and `#!c long double` are supported for floating-point numbers. !!! example diff --git a/docs/mkdocs/docs/features/types/template_parameters.md b/docs/mkdocs/docs/features/types/template_parameters.md index f20102878..888590234 100644 --- a/docs/mkdocs/docs/features/types/template_parameters.md +++ b/docs/mkdocs/docs/features/types/template_parameters.md @@ -30,9 +30,9 @@ Requirements are split into two groups: diagnosed with dedicated error messages, and violating most of them results in a compiler error somewhere inside the library. Four violations are not caught at compile time at all: - - A [`StringType`](#stringtype) whose `data()` is not null-terminated compiles and can silently misparse - floating-point numbers, because the lexer may hand the buffer to `#!cpp std::strtod`, which reads up to the - terminating null character. + - A [`StringType`](#stringtype) whose `data()` is not null-terminated compiles and silently misparses numbers + stored as a `#!cpp long double` that is not IEEE 754 binary64 (e.g., the 80-bit x87 format), because the lexer + hands the buffer to `#!cpp std::strtold`. - A stateful [`AllocatorType`](#allocatortype) compiles and silently ignores its state: allocation, deallocation, and [`get_allocator()`](../../api/basic_json/get_allocator.md) each use a different default-constructed instance. - The two [cross-specialization conversions](#cross-specialization-conversions) below. These abort on an assertion @@ -40,19 +40,19 @@ Requirements are split into two groups: ## Overview -| Template parameter | Default | Notable substitutes | -|-------------------------------------------------------------------|-----------------------------------|-----------------------------------------------------------------------| -| [`ObjectType`](#objecttype) | `std::map` | [`nlohmann::ordered_map`](../../api/ordered_map.md), Abseil hash maps | -| [`ArrayType`](#arraytype) | `std::vector` | `#!cpp std::deque` | -| [`StringType`](#stringtype) | `std::string` | `std::string`-like types over `char` | -| [`BooleanType`](#booleantype) | `bool` | none worth using | -| [`NumberIntegerType`](#numberintegertype-and-numberunsignedtype) | `std::int64_t` | any signed integer type | -| [`NumberUnsignedType`](#numberintegertype-and-numberunsignedtype) | `std::uint64_t` | any unsigned integer type at least as wide as `NumberIntegerType` | -| [`NumberFloatType`](#numberfloattype) | `double` | `float` (`long double`: no binary formats) | -| [`AllocatorType`](#allocatortype) | `std::allocator` | stateless allocators | -| [`JSONSerializer`](#jsonserializer) | `adl_serializer` | serializers with the same interface | -| [`BinaryType`](#binarytype) | `#!cpp std::vector` | `#!cpp std::vector` | -| [`CustomBaseClass`](#custombaseclass) | `void` | any default-constructible class | +| Template parameter | Default | Notable substitutes | +|-------------------------------------------------------------------|-----------------------------------|--------------------------------------------------------------------------------------------------| +| [`ObjectType`](#objecttype) | `std::map` | [`nlohmann::ordered_map`](../../api/ordered_map.md), Abseil hash maps | +| [`ArrayType`](#arraytype) | `std::vector` | `#!cpp std::deque` | +| [`StringType`](#stringtype) | `std::string` | `std::string`-like types over `char` | +| [`BooleanType`](#booleantype) | `bool` | none worth using | +| [`NumberIntegerType`](#numberintegertype-and-numberunsignedtype) | `std::int64_t` | any signed integer type | +| [`NumberUnsignedType`](#numberintegertype-and-numberunsignedtype) | `std::uint64_t` | any unsigned integer type at least as wide as `NumberIntegerType`, or `NumberIntegerType` itself | +| [`NumberFloatType`](#numberfloattype) | `double` | `float` (`long double`: no binary formats) | +| [`AllocatorType`](#allocatortype) | `std::allocator` | stateless allocators | +| [`JSONSerializer`](#jsonserializer) | `adl_serializer` | serializers with the same interface | +| [`BinaryType`](#binarytype) | `#!cpp std::vector` | `#!cpp std::vector` | +| [`CustomBaseClass`](#custombaseclass) | `void` | any default-constructible class | !!! warning "Third-party containers and incomplete types" @@ -353,16 +353,21 @@ using array_t = ArrayType>; ### Always required - A member type `value_type` that is one byte wide and `char`-compatible. The library stores and processes UTF-8 - encoded `char` data and passes `data()` to functions that take a `#!cpp const char*`, such as `#!cpp std::strtod`. + encoded `char` data and passes `data()` to functions that take a `#!cpp const char*`, such as `#!cpp std::strtold` + (only used to parse a `#!cpp long double` that is not IEEE 754 binary64, see + [`NumberFloatType`](#numberfloattype)). `#!cpp std::wstring`, `#!cpp std::u16string`, and `#!cpp std::u32string` are **not** valid choices; see the FAQ on [wide string handling](../../home/faq.md#wide-string-handling). - Constructors: default, copy, move, from `#!cpp const char*` (which must not be `#!cpp explicit`), from `#!cpp (const char*, size_type)`, and from `#!cpp (size_type, char)`; and copy or move assignment. - Member functions `size()`, `clear()`, `resize(n, c)`, `data()`, `push_back(char)`, and `operator[]` (const and non-const, returning references). `c_str()` and `back()` are **not** required. -- `data()` must return a pointer to a contiguous, **null-terminated** buffer -- the parser may hand it to - `#!cpp std::strtod`, which reads up to the null character. A type whose `data()` is not null-terminated does not - fail to compile; it can silently misparse floating-point numbers. +- `data()` must return a pointer to a contiguous, **null-terminated** buffer. `#!cpp float`, `#!cpp double`, and a + `#!cpp long double` that is IEEE 754 binary64 are converted by the library itself and do not depend on this. For any + other `NumberFloatType` (a `#!cpp long double` of another format), the parser falls back to `#!cpp std::strtold` when + `#!cpp std::from_chars` is not available or declines the token, and `std::strtold` reads up to the null character. A type whose `data()` + is not null-terminated does not fail to compile; with such a `NumberFloatType` it can silently misparse + floating-point numbers. - `append(const char*, size_type)`, used by [`dump`](../../api/basic_json/dump.md), and `append(const StringType&)`, used by the CBOR reader for indefinite-length strings. The library's internal string concatenation additionally has to append a `#!cpp char` and a `#!cpp const char*`; for each it selects between `append(arg)`, `#!cpp operator+=`, @@ -499,13 +504,14 @@ Both types are stored **directly** inside `basic_json`'s union. ### Always required - `#!cpp std::is_integral` must be satisfied: `NumberIntegerType` must be a **signed** integer type, - `NumberUnsignedType` an **unsigned** integer type. Class types are not supported -- among others, the constructors - taking integer values are constrained on `#!cpp std::is_integral`. + `NumberUnsignedType` an **unsigned** integer type or the same type as `NumberIntegerType` (see + [below](#same-type-for-both)). Class types are not supported -- among others, the constructors taking integer values + are constrained on `#!cpp std::is_integral`. - Trivially default-constructible, trivially copyable, and trivially destructible (union member). - `#!cpp std::numeric_limits` must be specialized for both types. -- `NumberUnsignedType` must be able to represent the absolute value of every `NumberIntegerType` value; serialization - of negative numbers converts the value to `NumberUnsignedType`. A `#!cpp static_assert` requires it to be at least as - wide as `NumberIntegerType`, which is what that amounts to for the standard integer types. +- An unsigned `NumberUnsignedType` must be able to represent the absolute value of every `NumberIntegerType` value; + serialization of negative numbers converts the value to `NumberUnsignedType`. A `#!cpp static_assert` requires it to + be at least as wide as `NumberIntegerType`, which is what that amounts to for the standard integer types. - Both types must fit into the internal 64-character number buffer used by [`dump`](../../api/basic_json/dump.md), which is the case for all standard integer types. - [`std::hash`](../../api/basic_json/std_hash.md) additionally requires `#!cpp std::hash` specializations. @@ -517,6 +523,23 @@ is stored as [`number_float_t`](../../api/basic_json/number_float_t.md) instead. therefore silently changes parse results rather than raising an error. See [Number Handling](number_handling.md) for details. +#### Same type for both + +`NumberUnsignedType` may be the same type as `NumberIntegerType`, e.g. +`#!cpp nlohmann::json::with_integers_t` (see +[`with_integers_t`](../../api/basic_json/with_t.md)), for code that only wants to handle one integer type. Such a `basic_json` stores every integer as +[`number_integer_t`](../../api/basic_json/number_integer_t.md): + +- [`is_number_unsigned()`](../../api/basic_json/is_number_unsigned.md) is always `#!cpp false`, and + [`type()`](../../api/basic_json/type.md) never returns `value_t::number_unsigned`. Constructing a value from + `value_t::number_unsigned` gives a `value_t::number_integer` value. +- Values of unsigned C++ types (e.g., `#!cpp 5u`) are converted to `NumberIntegerType`, so values above its maximum + wrap around, as values outside the range of a narrower `NumberIntegerType` already do. +- The parsers store a non-negative integer above the maximum of `NumberIntegerType` (e.g., `18446744073709551615` for + `#!cpp std::int64_t`) as [`number_float_t`](../../api/basic_json/number_float_t.md), like any other integer that does + not fit. +- Converting from a `basic_json` with distinct types turns its unsigned values into `number_integer_t` values. + ### Compatible types | Type pair | Support | @@ -524,8 +547,9 @@ therefore silently changes parse results rather than raising an error. See | `#!cpp std::int64_t` / `#!cpp std::uint64_t` (default) | full | | `#!cpp std::int32_t` / `#!cpp std::uint32_t`, `#!cpp long long` / `#!cpp unsigned long long` | full; narrower types change which literals the parser can represent | | any other pair of standard signed/unsigned integer types | full | +| the same signed integer type for both, e.g. `#!cpp std::int64_t` / `#!cpp std::int64_t` | full; every integer is stored as `number_integer_t`, see [above](#same-type-for-both) | | class types, enumerations | not usable; `#!cpp std::is_integral` must hold | -| `#!cpp bool`, or a type already used for another member of the union | not usable; `#!cpp std::is_integral` is in fact `#!cpp true`, but the `get_impl_ptr` overloads for `boolean_t`, `number_integer_t`, `number_unsigned_t` and `number_float_t` would collide | +| `#!cpp bool`, `number_float_t`, or the type of another member of the union | not usable; `#!cpp std::is_integral` is in fact `#!cpp true`, but the `get_impl_ptr` overloads for `boolean_t`, `number_integer_t`, `number_unsigned_t` and `number_float_t` would collide | ## `NumberFloatType` @@ -541,16 +565,18 @@ therefore silently changes parse results rather than raising an error. See `NumberFloatType` must be one of `#!cpp float`, `#!cpp double`, or `#!cpp long double`: -- The [parser](../parsing/index.md) converts number literals with `#!cpp std::from_chars` or, as a fallback, with - `#!cpp std::strtof`, `#!cpp std::strtod`, or `#!cpp std::strtold`; the library provides overloads for exactly these - three types. +- The [parser](../parsing/index.md) converts number literals to `#!cpp float`, `#!cpp double`, and a + `#!cpp long double` that is IEEE 754 binary64 itself; other `#!cpp long double` formats are converted with + `#!cpp std::from_chars` where available, or with `#!cpp std::strtold`. The library provides overloads for exactly + these three types. - [`dump`](../../api/basic_json/dump.md) falls back to `#!cpp std::snprintf` with the `%g` and `%Lg` conversion specifiers, for which the library likewise provides only `#!cpp double` and `#!cpp long double` overloads (`#!cpp float` is promoted to `#!cpp double`). -If `#!cpp std::numeric_limits` describes an IEEE 754 binary32 or binary64 number, `dump` uses the -Grisu2 algorithm, which produces the shortest representation that round-trips. Otherwise the `snprintf` fallback with -`max_digits10` digits is used. +If `#!cpp std::numeric_limits` describes an IEEE 754 binary64 number, `dump` uses the algorithm of +Żmij, which produces the shortest representation that round-trips. For IEEE 754 binary32 numbers, it uses Grisu2, +which produces a short representation that round-trips. Otherwise the `snprintf` fallback with `max_digits10` digits is +used. ### Required for the binary formats @@ -562,7 +588,7 @@ binary32 or binary64 field and have no encoding for `#!cpp long double`. | Type | Support | |--------------------------|-----------------------------------------------------------------------------------------------------------------------| -| `#!cpp double` (default) | full; short round-trip output through Grisu2 | +| `#!cpp double` (default) | full; shortest round-trip output through Żmij | | `#!cpp float` | full; short round-trip output through Grisu2 | | `#!cpp long double` | `dump` and `parse` only; the binary format writers do not compile, as they only handle IEEE 754 binary32 and binary64 | | any other type | not usable | diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index ea47bab6e..4975c1d1b 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -185,9 +185,11 @@ The lexer detects two optional extensions at compile time. Only `iterator_input_ random-access input of single-byte characters: - `supports_seek`, `get_consumed_count()`, and `copy_consumed_range()` let the lexer reconstruct already consumed input - for error messages instead of copying every character it reads. + for error messages instead of copying every character it reads. With `get_read_count()` and + `count_consumed_lines()`, the lexer also derives the read position from the adapter and counts lines only when an + error needs them, instead of counting characters and lines on every read. - `supports_bulk_scan`, `bulk_data()`, `bulk_remaining()`, and `bulk_skip()` let the lexer scan strings directly in - contiguous memory, several bytes at a time. + contiguous memory, several bytes at a time, and skip the spaces that indent a line in 8-byte steps. The function `input_adapter` picks the right adapter for the argument passed to `parse`, `accept`, `sax_parse`, or the `from_*` functions: diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 02fd85689..a7e935538 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -889,7 +889,9 @@ contains other characters (e.g., `1a`), or `-` where it cannot be used. ### json.exception.out_of_range.405 -The JSON Patch operations 'remove' and 'add' cannot be applied to the root element of the JSON value. +The JSON Patch operations 'remove' and 'move' cannot be applied to the root element of the JSON value. The exception is +also thrown when the parent or a reference token of an empty JSON Pointer is requested, for instance by +[`back`](../api/json_pointer/back.md) or [`pop_back`](../api/json_pointer/pop_back.md). !!! failure "Example message" @@ -914,15 +916,23 @@ double-precision number when `number_float_t` is `#!cpp float`. ### json.exception.out_of_range.407 -This exception previously indicated that the UBJSON and BSON binary formats did not support integer numbers greater than -9223372036854775807 due to limitations in the implemented mapping. However, these limitations have since been resolved, -and this exception no longer occurs. +An unsigned integer number greater than 9223372036854775807 cannot be represented in the binary format. This exception +is only thrown by [`to_bon8`](../api/basic_json/to_bon8.md), as [BON8](../features/binary_formats/bon8.md) has no +unsigned 64-bit integer type. -!!! success "Exception cannot occur any more" +!!! note "Former occurrences" + + The UBJSON and BSON writers used to throw this exception as well: - Since version 3.9.0, integer numbers beyond int64 are serialized as high-precision UBJSON numbers. - Since version 3.12.0, integer numbers beyond int64 are serialized as uint64 BSON numbers. +!!! failure "Example message" + + ``` + integer number 18446744073709551615 cannot be represented by BON8 as it does not fit int64 + ``` + ### json.exception.out_of_range.408 The size of an array or object in a [binary format](../features/binary_formats/index.md) exceeds the maximal capacity: diff --git a/docs/mkdocs/docs/home/faq.md b/docs/mkdocs/docs/home/faq.md index eec13a481..2381f4f7e 100644 --- a/docs/mkdocs/docs/home/faq.md +++ b/docs/mkdocs/docs/home/faq.md @@ -66,7 +66,9 @@ Without the macro (default behavior), `json j{obj}` creates `[{"key":"value"}]`. Can you add an option to ignore trailing commas? -This library does not support any feature that would jeopardize interoperability. +Yes, trailing commas in arrays and objects can be allowed by setting parameter `ignore_trailing_commas` of +[`parse`](../api/basic_json/parse.md) to `#!cpp true`; see [trailing commas](../features/trailing_commas.md). They are +rejected by default, and the library never adds them when serializing. ### Parse errors reading non-ASCII characters @@ -230,11 +232,11 @@ Yes, you can. Please define the symbol [`JSON_DIAGNOSTICS`](../api/macros/json_d - It seems that precision is lost when serializing a double. - Can I change the precision for floating-point serialization? -The library uses `std::numeric_limits::digits10` (15 for IEEE `double`s) digits for serialization. This value is sufficient to guarantee roundtripping. If one uses more than this number of digits of precision, then string -> value -> string is not guaranteed to round-trip. +The library serializes floating-point numbers with the shortest representation that parses back to the same value, using at most `std::numeric_limits::max_digits10` (17 for IEEE `double`s) significant digits. This guarantees that value -> string -> value round-trips. The reverse direction, string -> value -> string, is only guaranteed for numbers with at most `std::numeric_limits::digits10` (15 for IEEE `double`s) significant digits; a number with more digits may not be representable exactly and is then serialized as the closest `double`. -!!! quote "[cppreference.com](https://en.cppreference.com/w/cpp/types/numeric_limits/digits10)" +!!! quote "[cppreference.com](https://en.cppreference.com/w/cpp/types/numeric_limits/max_digits10)" - The value of `std::numeric_limits::digits10` is the number of base-10 digits that can be represented by the type T without change, that is, any number with this many significant decimal digits can be converted to a value of type T and back to decimal form, without change due to rounding or overflow. + The value of `std::numeric_limits::max_digits10` is the number of base-10 digits that are necessary to uniquely represent all distinct values of the type T, such as necessary for serialization/deserialization to text. !!! tip diff --git a/docs/mkdocs/docs/home/license.md b/docs/mkdocs/docs/home/license.md index 11d0ec529..715c6ca51 100644 --- a/docs/mkdocs/docs/home/license.md +++ b/docs/mkdocs/docs/home/license.md @@ -18,6 +18,8 @@ The class contains the UTF-8 Decoder from Bjoern Hoehrmann which is licensed und The class contains a slightly modified version of the Grisu2 algorithm from Florian Loitsch which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2009 [Florian Loitsch](https://florian.loitsch.com/) +The class contains a port of the shortest double-to-decimal conversion of [Żmij](https://github.com/vitaut/zmij) by Victor Zverovich, including the conversion of the digits to text by Xiang JunBo and the SIMD instruction sequence of Dougall Johnson, which is licensed under the [MIT License](https://opensource.org/licenses/MIT) (see above). Copyright © 2025 [Victor Zverovich](https://github.com/vitaut) + The class contains a copy of [Hedley](https://nemequ.github.io/hedley/) from Evan Nemerson which is licensed as [CC0-1.0](https://creativecommons.org/publicdomain/zero/1.0/). -The class contains an adapted version of the Eisel-Lemire algorithm and its table of powers of five from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors +The class contains an adapted version of the Eisel-Lemire algorithm, its table of powers of five, and its digit comparison for long numbers from [fast_float](https://github.com/fastfloat/fast_float) by Daniel Lemire and contributors, which is available under the [MIT License](https://opensource.org/licenses/MIT) (used here), the Apache 2.0 License, and the Boost Software License. Copyright © 2021 The fast_float authors diff --git a/docs/mkdocs/docs/home/releases.md b/docs/mkdocs/docs/home/releases.md index fabf95e8a..c0865270e 100644 --- a/docs/mkdocs/docs/home/releases.md +++ b/docs/mkdocs/docs/home/releases.md @@ -9,6 +9,15 @@ published on the [GitHub releases page](https://github.com/nlohmann/json/release This documentation is built from the `develop` branch and may describe changes that are not part of a release yet. Their version numbers are followed by an unreleased badge. +## Unreleased + +- Converting an rvalue container to JSON (`json j = std::move(vec);`, `json j = std::map{...};`) + now **moves** the elements instead of copying them (e.g., `std::vector` with 100,000 strings: 4.7 ms to + 2.3 ms). Two consequences: the source container is left in a valid but unspecified (moved-from) state, and if an + exception is thrown during the conversion, the source may be partially moved-from, so the strong exception guarantee + no longer holds for the source of an rvalue conversion (the JSON value itself is unaffected). Lvalues and `const` + objects are still copied. See [`basic_json`](../api/basic_json/basic_json.md). + ## v3.12.0 (2025-04-11) Fixes bugs found in 3.11.3 and adds several features. All changes are backward-compatible. diff --git a/docs/mkdocs/docs/integration/cmake.md b/docs/mkdocs/docs/integration/cmake.md index 9492efe9f..65eed72a8 100644 --- a/docs/mkdocs/docs/integration/cmake.md +++ b/docs/mkdocs/docs/integration/cmake.md @@ -90,13 +90,13 @@ to the following. ### FetchContent -Since CMake v3.11, [FetchContent](https://cmake.org/cmake/help/v3.11/module/FetchContent.html) can be used to +Since CMake v3.14, [FetchContent](https://cmake.org/cmake/help/v3.14/module/FetchContent.html) can be used to automatically download a release as a dependency at configure time. !!! example ```cmake title="CMakeLists.txt" - cmake_minimum_required(VERSION 3.11) + cmake_minimum_required(VERSION 3.14) project(ExampleProject LANGUAGES CXX) include(FetchContent) diff --git a/docs/mkdocs/docs/integration/compile_times.md b/docs/mkdocs/docs/integration/compile_times.md index 0e3be638c..279a00ec7 100644 --- a/docs/mkdocs/docs/integration/compile_times.md +++ b/docs/mkdocs/docs/integration/compile_times.md @@ -14,7 +14,8 @@ that cost, ordered by how much they typically save. Header files that only need to *name* the `json` type — for function declarations, members held by pointer or reference, or friend declarations — can include `` instead of ``. It only -forward-declares `basic_json`, `json`, `ordered_json`, `json_pointer`, and `adl_serializer`. The translation units that +forward-declares `basic_json`, `json_pointer`, `adl_serializer`, and `ordered_map`, and defines the aliases `json` and +`ordered_json`. The translation units that actually use the values then include ``. ```cpp title="person.hpp" diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 69775c876..c183a5296 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -183,6 +183,7 @@ nav: - 'json_serializer': api/basic_json/json_serializer.md - 'max_size': api/basic_json/max_size.md - 'meta': api/basic_json/meta.md + - 'merge_diff': api/basic_json/merge_diff.md - 'merge_patch': api/basic_json/merge_patch.md - 'number_float_t': api/basic_json/number_float_t.md - 'number_integer_t': api/basic_json/number_integer_t.md @@ -453,8 +454,12 @@ plugins: # package badges (the fetch fails, then reading the missing cache entry # raises FileNotFoundError). Readers' browsers are served normally, so # leave these badges as external references instead of self-hosting them. + # The same goes for img.shields.io, whose version badges time out from + # time to time (strict mode turns the failed download into an error) and + # would go stale between builds if they were self-hosted anyway. assets_exclude: - repology.org/* + - img.shields.io/* - llmstxt: markdown_description: > JSON for Modern C++ is a C++11 header-only library implementing a JSON diff --git a/docs/mkdocs/requirements.txt b/docs/mkdocs/requirements.txt index d9e7098ed..245ebc387 100644 --- a/docs/mkdocs/requirements.txt +++ b/docs/mkdocs/requirements.txt @@ -7,6 +7,6 @@ mkdocs-material-extensions==1.3.1 # extensions mkdocs-minify-plugin==0.8.0 # plugin "minify" mkdocs-redirects==1.2.3 # plugin "redirects" mkdocs-htmlproofer-plugin==1.6.0 # plugin "htmlproofer" -mkdocs-llmstxt==0.5.0 # plugin "llmstxt" +mkdocs-llmstxt==0.5.1 # plugin "llmstxt" PyYAML==6.0.3 # linter diff --git a/docs/mkdocs/scripts/mermaid/package-lock.json b/docs/mkdocs/scripts/mermaid/package-lock.json index 4f9dc0395..d7d3cad06 100644 --- a/docs/mkdocs/scripts/mermaid/package-lock.json +++ b/docs/mkdocs/scripts/mermaid/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "check-mermaid", "dependencies": { - "jsdom": "30.1.1", + "jsdom": "30.1.2", "mermaid": "11.17.2" } }, @@ -24,14 +24,14 @@ } }, "node_modules/@asamuzakjp/css-color": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-7.1.2.tgz", - "integrity": "sha512-99DHAnXDB5z6EEK+9GMpVI7Mw4oxj97dY5bpOzMnjADQWxI8rN6TvTduuFLUhUMlS7/CfVZ06tcZsus6cltnNw==", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-7.1.3.tgz", + "integrity": "sha512-1t1U8Cm3RBl9vh3RAZhjkPaduzLSvQJRToRrxpGR/eDRW99mOREleuu+mWfiFa795bUzjwht7NgphCKTs2ianQ==", "license": "MIT", "dependencies": { - "@csstools/css-calc": "^3.4.1", - "@csstools/css-color-parser": "^4.2.4", - "@csstools/css-parser-algorithms": "^4.0.1", + "@csstools/css-calc": "^3.4.2", + "@csstools/css-color-parser": "^4.2.5", + "@csstools/css-parser-algorithms": "^4.0.2", "@csstools/css-tokenizer": "^4.0.2", "lru-cache": "^11.5.3" }, @@ -40,9 +40,9 @@ } }, "node_modules/@asamuzakjp/dom-selector": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-9.2.2.tgz", - "integrity": "sha512-lSWTBMjAcmu2xn5yEDU7jh6QDV+C8GEKtdJ4pIQhXh26RkKQ7S3FuQlt+zZkUbTdJ4d3XyV1kPI/G0zWXxqaqw==", + "version": "9.2.4", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-9.2.4.tgz", + "integrity": "sha512-YnVzxDxLaqL0t7Q4wfcgHZjg55Wm6y5EaO7Bhu3q9J7wq/8iw3PhsnWiOQhYekx4VexnbkgWpTT6n+QIV7kczw==", "license": "MIT", "dependencies": { "bidi-js": "^1.1.0", @@ -98,9 +98,9 @@ } }, "node_modules/@csstools/css-calc": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.4.1.tgz", - "integrity": "sha512-EtC7SoN1j6J4E4DCwg5QgbO5TGxgxIA1RXqe+W+qUM+BUcezx9wT+/tiQ/WO2yCX4i5X+Cuf9ciJ22aP4UEwWw==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.4.3.tgz", + "integrity": "sha512-iex20d8CHVkyvg6B7UKV7uHnI2Bqo9g+EFfT9E0y+GvTvhZ/DwONJ+9aKb1dlqm0ZiGsL5RXjp0fCoJYnkeDjA==", "funding": [ { "type": "github", @@ -116,14 +116,14 @@ "node": ">=20.19.0" }, "peerDependencies": { - "@csstools/css-parser-algorithms": "^4.0.1", + "@csstools/css-parser-algorithms": "^4.0.2", "@csstools/css-tokenizer": "^4.0.2" } }, "node_modules/@csstools/css-color-parser": { - "version": "4.2.4", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.2.4.tgz", - "integrity": "sha512-DyefytAZ735mX4Dq/WcDAXFtXhaEFvme0ZS9tVEBAc2whxUthXr0R0L2rmEPm59SrMBkGrFzQviBXMs/UtnABQ==", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.2.6.tgz", + "integrity": "sha512-iiPQ3iRWwnJkeEn6RIu6SJPr7hYrLz6XZ9s/QZl+2/LI5KQVjpl2fdmDSZKuD4xP6GMmMPgHFFXg6k1Wkz0Trg==", "funding": [ { "type": "github", @@ -137,20 +137,20 @@ "license": "MIT", "dependencies": { "@csstools/color-helpers": "^6.1.2", - "@csstools/css-calc": "^3.4.1" + "@csstools/css-calc": "^3.4.3" }, "engines": { "node": ">=20.19.0" }, "peerDependencies": { - "@csstools/css-parser-algorithms": "^4.0.1", + "@csstools/css-parser-algorithms": "^4.0.2", "@csstools/css-tokenizer": "^4.0.2" } }, "node_modules/@csstools/css-parser-algorithms": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.1.tgz", - "integrity": "sha512-ShL8BqPfbKJrJiKFH0xBbN0i7Nrh9HXYRuF+pzyj93R/BL2YAsUxJeqANErzqM+0JGl7vjHp+3OIgd/DL69YIA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.2.tgz", + "integrity": "sha512-40cSKyMvK+tq4qz6Awrlye2WGuOKt3FwPgtGg6KTfbHOWNw+Rk1rzbAtZnZ6IBhsY491HLRnDXwoyBAijmmILA==", "funding": [ { "type": "github", @@ -170,9 +170,9 @@ } }, "node_modules/@csstools/css-syntax-patches-for-csstree": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.14.tgz", - "integrity": "sha512-HpbVXyrofRXpHpgkNIjU/3EWR4WJvOkO3emNK/L6X/mTJU7bGUI3AkkpoTNXznQLp0KRjLHELTGeKI5dIkI9JQ==", + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.15.tgz", + "integrity": "sha512-J0u7HkVl2nzSlhsiTOp4AmwcUQ3D+mGEEKfBy/7To5/y7F2OHwyLrXfrhR0SMgr4p5Lo+eaMVSeai24zUcBIxA==", "funding": [ { "type": "github", @@ -1071,30 +1071,16 @@ } }, "node_modules/data-urls": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", - "integrity": "sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-8.0.0.tgz", + "integrity": "sha512-JQUCCK+/CAGbGGx618PlFZMXH81Jtj7t73CHv9dd/zVWvxgGT2zLFRjnKb+Ng46GIwcy+f7f88qd4duqllK+WQ==", "license": "MIT", "dependencies": { "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^16.0.0" + "whatwg-url": "^17.1.2" }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/data-urls/node_modules/whatwg-url": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", - "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", - "license": "MIT", - "dependencies": { - "@exodus/bytes": "^1.11.0", - "tr46": "^6.0.0", - "webidl-conversions": "^8.0.1" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "node": "^22.14.0 || >=24.0.0" } }, "node_modules/dayjs": { @@ -1216,30 +1202,30 @@ "license": "MIT" }, "node_modules/jsdom": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.1.1.tgz", - "integrity": "sha512-FahmoPK5vbPc+jxV1iErMHmAZypCZ942NHF4+qqaWAuvaKKTBZxawnmAtrbGWLU7MtlxfqIP0qw6aSI+aWGtLg==", + "version": "30.1.2", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.1.2.tgz", + "integrity": "sha512-0FFE/jE1rppmVfUrJUgxqXjcwolZYIGtAgj1pTussMhNZxIxi7W/PvfWaMNroGi2B36X1IKHbexpZ9DhkoOPiQ==", "license": "MIT", "dependencies": { - "@asamuzakjp/css-color": "^7.0.0", - "@asamuzakjp/dom-selector": "^9.2.1", + "@asamuzakjp/css-color": "^7.1.3", + "@asamuzakjp/dom-selector": "^9.2.4", "@bramus/specificity": "^2.4.2", - "@csstools/css-syntax-patches-for-csstree": "^1.1.13", - "@exodus/bytes": "^1.15.1", + "@csstools/css-syntax-patches-for-csstree": "^1.1.15", + "@exodus/bytes": "^1.16.0", "css-tree": "^3.2.1", - "data-urls": "^7.0.0", + "data-urls": "^8.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^7.0.0", "is-potential-custom-element-name": "^1.0.1", - "lru-cache": "^11.5.2", + "lru-cache": "^11.5.3", "parse5": "^8.0.1", "saxes": "^6.0.0", "tough-cookie": "^6.0.2", - "undici": "^8.10.2", + "undici": "^8.11.2", "w3c-xmlserializer": "^6.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^17.1.1", + "whatwg-url": "^17.2.0", "xml-name-validator": "^5.0.0" }, "engines": { @@ -1514,15 +1500,15 @@ } }, "node_modules/tr46": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-6.0.0.tgz", - "integrity": "sha512-bLVMLPtstlZ4iMQHpFHTR7GAGj2jxi8Dg0s2h2MafAE4uSWF98FC/3MomU51iQAMf8/qDUbKWf5GxuvvVcXEhw==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-7.0.0.tgz", + "integrity": "sha512-BcDSDW+SIGWVOns1Zm6Yg91q9AEj7eaID0QDY9oIqJD1A8dscNwMcl9Sms2DeuWCNov1X0/oXanHGYQBSLwZsg==", "license": "MIT", "dependencies": { "punycode": "^2.3.1" }, "engines": { - "node": ">=20" + "node": "^22.14.0 || >=24.0.0" } }, "node_modules/ts-dedent": { @@ -1587,13 +1573,13 @@ } }, "node_modules/whatwg-url": { - "version": "17.1.2", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-17.1.2.tgz", - "integrity": "sha512-TEZA+Zqxin7Jjsm2cjRohCmen5awh+hT6Zi3VZdqZlNRk7zvOI/9WpBFg/DWlA56bWnzwm6DuB8NS0EsxQH9uQ==", + "version": "17.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-17.2.0.tgz", + "integrity": "sha512-IhLIHNcBrzIixC5XXsbr4DxdjajyA5vQ5RwGZMtrUp+GbtE5Nrsge9QKau7jXrC0aTXLYGV0xiW7Tb4YS2aomw==", "license": "MIT", "dependencies": { - "@exodus/bytes": "^1.15.1", - "tr46": "^6.0.0", + "@exodus/bytes": "^1.16.0", + "tr46": "^7.0.0", "webidl-conversions": "^8.0.1" }, "engines": { diff --git a/docs/mkdocs/scripts/mermaid/package.json b/docs/mkdocs/scripts/mermaid/package.json index 6cf40b9f6..c79f4fa89 100644 --- a/docs/mkdocs/scripts/mermaid/package.json +++ b/docs/mkdocs/scripts/mermaid/package.json @@ -4,7 +4,7 @@ "description": "Validate the Mermaid diagrams of the documentation (see check_mermaid.mjs)", "type": "module", "dependencies": { - "jsdom": "30.1.1", + "jsdom": "30.1.2", "mermaid": "11.17.2" } } diff --git a/include/nlohmann/detail/bit_ops.hpp b/include/nlohmann/detail/bit_ops.hpp index 9ccb82c66..c9424a6d1 100644 --- a/include/nlohmann/detail/bit_ops.hpp +++ b/include/nlohmann/detail/bit_ops.hpp @@ -9,11 +9,12 @@ #pragma once #include // uint64_t -#if !defined(__SIZEOF_INT128__) && defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) - #include // __umulh, _umul128 +#include // memcpy +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) && (!defined(__SIZEOF_INT128__) || (!defined(__GNUC__) && !defined(__clang__))) + #include // __umulh, _umul128, _BitScanForward64, _BitScanReverse64 #endif -#include +#include // JSON_HEDLEY_ALWAYS_INLINE, NLOHMANN_JSON_NAMESPACE_BEGIN // Portable bit-level helpers for the number and string scanners. They use // compiler builtins or platform-specific intrinsics where available and plain @@ -28,6 +29,10 @@ inline int count_leading_zeros(std::uint64_t x) noexcept { #if defined(__GNUC__) || defined(__clang__) return __builtin_clzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanReverse64(&index, x); + return 63 - static_cast(index); #else int n = 0; for (int shift = 32; shift != 0; shift >>= 1) @@ -42,6 +47,29 @@ inline int count_leading_zeros(std::uint64_t x) noexcept #endif } +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanForward64(&index, x); + return static_cast(index); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + /// the 128-bit product of two 64-bit numbers struct uint128_parts { @@ -75,15 +103,27 @@ inline uint128_parts full_multiplication(std::uint64_t a, std::uint64_t b) noexc #endif } -/// eight bytes as a little-endian word (compilers fold this into one load on -/// little-endian targets) -inline std::uint64_t read_eight_bytes(const char* p) noexcept +/// eight bytes as a little-endian word (a single load on little-endian +/// targets; always inlined, as GCC otherwise calls it in the number loops) +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const unsigned char* b) noexcept { - const auto* b = reinterpret_cast(p); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + // the byte order already matches (all MSVC targets are little-endian) + std::uint64_t result = 0; + std::memcpy(&result, b, sizeof(result)); + return result; +#else return static_cast(b[0]) | (static_cast(b[1]) << 8u) | (static_cast(b[2]) << 16u) | (static_cast(b[3]) << 24u) | (static_cast(b[4]) << 32u) | (static_cast(b[5]) << 40u) | (static_cast(b[6]) << 48u) | (static_cast(b[7]) << 56u); +#endif +} + +/// eight bytes as a little-endian word +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const char* p) noexcept +{ + return read_eight_bytes(reinterpret_cast(p)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) } } // namespace detail diff --git a/include/nlohmann/detail/conversions/from_json.hpp b/include/nlohmann/detail/conversions/from_json.hpp index 0332474b2..5cb123c59 100644 --- a/include/nlohmann/detail/conversions/from_json.hpp +++ b/include/nlohmann/detail/conversions/from_json.hpp @@ -80,7 +80,11 @@ void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) { case value_t::number_unsigned: { - val = static_cast(*j.template get_ptr()); + // never null: a basic_json whose number_unsigned_t is number_integer_t + // stores no number_unsigned values (#2573) + const auto* ptr = j.template get_ptr(); + JSON_ASSERT(ptr != nullptr); + val = static_cast(*ptr); break; } case value_t::number_integer: @@ -150,7 +154,7 @@ inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_flo } template -inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +inline void from_json(const BasicJsonType& j, number_unsigned_overload_t& val) { get_arithmetic_value(j, val); } @@ -266,6 +270,22 @@ template void from_json_array_reserve(ConstructibleArrayType& /*arr*/, std::size_t /*size*/, priority_tag<0> /*unused*/) {} +// append through push_back() where available; containers without it (such as +// std::set) fall back to insert(end(), ...), which is what std::inserter did +template +auto from_json_array_append(ConstructibleArrayType& arr, ValueType&& value, priority_tag<1> /*unused*/) +-> decltype(arr.push_back(std::forward(value)), void()) +{ + arr.push_back(std::forward(value)); +} + +template +void from_json_array_append(ConstructibleArrayType& arr, ValueType&& value, priority_tag<0> /*unused*/) +{ + using std::end; + arr.insert(end(arr), std::forward(value)); +} + template::value, @@ -275,17 +295,15 @@ auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, p j.template get(), void()) { - using std::end; - ConstructibleArrayType ret; - from_json_array_reserve(ret, j.size(), priority_tag<1> {}); - std::transform(j.begin(), j.end(), - std::inserter(ret, end(ret)), [](const BasicJsonType & i) + const auto* inner_array = j.template get_ptr(); + from_json_array_reserve(ret, inner_array->size(), priority_tag<1> {}); + for (const auto& elem : *inner_array) { // get() returns *this, this won't call a from_json // method when value_type is BasicJsonType - return i.template get(); - }); + from_json_array_append(ret, elem.template get(), priority_tag<1> {}); + } arr = std::move(ret); } @@ -371,6 +389,23 @@ template inline void from_json_object_reserve(ConstructibleObjectType& /*obj*/, std::size_t /*size*/, priority_tag<0> /*unused*/) {} +// the keys of the source are sorted for the usual object types, so hinting the +// end makes every insertion constant-time (for containers that order their keys +// the same way); containers without emplace_hint() (such as ordered_map) fall +// back to emplace() +template +auto from_json_object_emplace(ConstructibleObjectType& obj, Key&& key, Value&& value, priority_tag<1> /*unused*/) +-> decltype(obj.emplace_hint(obj.end(), std::forward(key), std::forward(value)), void()) +{ + obj.emplace_hint(obj.end(), std::forward(key), std::forward(value)); +} + +template +void from_json_object_emplace(ConstructibleObjectType& obj, Key&& key, Value&& value, priority_tag<0> /*unused*/) +{ + obj.emplace(std::forward(key), std::forward(value)); +} + template::value, int> = 0> inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) @@ -385,7 +420,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) from_json_object_reserve(ret, inner_object->size(), priority_tag<1> {}); for (const auto& p : *inner_object) { - ret.emplace(p.first, p.second.template get()); + from_json_object_emplace(ret, p.first, p.second.template get(), priority_tag<1> {}); } obj = std::move(ret); } @@ -406,7 +441,11 @@ inline void from_json(const BasicJsonType& j, ArithmeticType& val) { case value_t::number_unsigned: { - val = static_cast(*j.template get_ptr()); + // never null: a basic_json whose number_unsigned_t is number_integer_t + // stores no number_unsigned values (#2573) + const auto* ptr = j.template get_ptr(); + JSON_ASSERT(ptr != nullptr); + val = static_cast(*ptr); break; } case value_t::number_integer: diff --git a/include/nlohmann/detail/conversions/to_chars.hpp b/include/nlohmann/detail/conversions/to_chars.hpp index 52342db40..db50795b5 100644 --- a/include/nlohmann/detail/conversions/to_chars.hpp +++ b/include/nlohmann/detail/conversions/to_chars.hpp @@ -4,6 +4,7 @@ // |_____|_____|_____|_|___| https://github.com/nlohmann/json // // SPDX-FileCopyrightText: 2009 Florian Loitsch +// SPDX-FileCopyrightText: 2025 Victor Zverovich // SPDX-FileCopyrightText: 2013-2026 Niels Lohmann // SPDX-License-Identifier: MIT @@ -11,11 +12,32 @@ #include // array #include // signbit, isfinite +#include // size_t #include // intN_t, uintN_t #include // memcpy, memmove #include // numeric_limits #include // conditional +#ifdef _MSC_VER + #include // _byteswap_uint64 +#endif + +// SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 +// digits of a double at once +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) + #include + #define JSON_DTOA_SSE2 1 + #define JSON_DTOA_NEON 0 +#elif (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_M_ARM64EC) && !defined(__ARM_BIG_ENDIAN) + #include + #define JSON_DTOA_SSE2 0 + #define JSON_DTOA_NEON 1 +#else + #define JSON_DTOA_SSE2 0 + #define JSON_DTOA_NEON 0 +#endif + +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -1047,6 +1069,381 @@ inline char* format_buffer(char* buf, int len, int decimal_exponent, return append_exponent(buf, n - 1); } +/// eight decimal digits (a value below 10^8) as bytes 0..9, the first digit +/// in the most significant byte: three steps that divide all lanes at once +/// by a multiplication (the conversion of Xiang JunBo, as in Zmij) +inline std::uint64_t eight_digit_bytes(std::uint64_t abcdefgh) noexcept +{ + const std::uint64_t abcd_efgh = abcdefgh + (((std::uint64_t{1} << 32u) - 10000u) * ((abcdefgh * (((std::uint64_t{1} << 40u) / 10000u) + 1u)) >> 40u)); + const std::uint64_t ab_cd_ef_gh = abcd_efgh + (((std::uint64_t{1} << 16u) - 100u) * (((abcd_efgh * (((std::uint64_t{1} << 19u) / 100u) + 1u)) >> 19u) & 0x7F0000007Fu)); + return ab_cd_ef_gh + (((std::uint64_t{1} << 8u) - 10u) * (((ab_cd_ef_gh * (((std::uint64_t{1} << 10u) / 10u) + 1u)) >> 10u) & 0x000F000F000F000Fu)); +} + +/// store the bytes of v, the most significant one first (one byte swap and +/// one store where the byte order is known: compilers do not reliably merge +/// the byte stores once this is inlined) +inline void store_msb_first(char* p, std::uint64_t v) noexcept +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + v = __builtin_bswap64(v); + std::memcpy(p, &v, sizeof(v)); +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + std::memcpy(p, &v, sizeof(v)); +#elif defined(_MSC_VER) // (little-endian on all its targets) + v = _byteswap_uint64(v); + std::memcpy(p, &v, sizeof(v)); +#else + for (unsigned i = 0; i < 8; ++i) + { + p[i] = static_cast(v >> (56u - (8u * i))); + } +#endif +} + +/*! +@brief digits * 10^exp for a double, in the layout of format_buffer() + +The layout is that of format_buffer() with min_exp -4 and max_exp 15 (the +digits10 of double). The digits are converted eight at a time and placed +with fixed-size moves instead of per-digit loops and moves of the buffer. + +@param[in] digits the digits (not 0, at most 17 digits; trailing zeros allowed) +@param[in] exp the decimal exponent of the last digit +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_decimal(char* first, std::uint64_t digits, int exp) noexcept +{ + JSON_ASSERT(digits != 0 && digits < 100000000000000000u); + const std::uint64_t upper = digits / 100000000u; + const std::uint64_t b0 = upper / 100000000u; // (one digit: it is its own byte) + const std::uint64_t b1 = eight_digit_bytes(upper % 100000000u); + const std::uint64_t b2 = eight_digit_bytes(digits % 100000000u); + // leading and trailing zero digits: zero bytes, counted without division + int leading = 16; + int zeros = 16; + if (b0 != 0) + { + leading = count_leading_zeros(b0) / 8; + } + else if (b1 != 0) + { + leading = 8 + (count_leading_zeros(b1) / 8); + } + else + { + leading += count_leading_zeros(b2) / 8; + } + if (b2 != 0) + { + zeros = count_trailing_zeros(b2) / 8; + } + else if (b1 != 0) + { + zeros = 8 + (count_trailing_zeros(b1) / 8); + } + // (else: 16, b0 is the one digit that is not 0) + // the digits as text at text + leading, then '0's, so that fixed-size + // moves need not check how many digits there are + std::array text; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + store_msb_first(text.data(), b0 + 0x3030303030303030u); + store_msb_first(text.data() + 8, b1 + 0x3030303030303030u); + store_msb_first(text.data() + 16, b2 + 0x3030303030303030u); + std::memset(text.data() + 24, '0', 40); + const int k = 24 - leading - zeros; // significant digits + const int n = k + exp + zeros; // position of the decimal point after the first digit + const char* const s0 = text.data() + leading; + + if (-4 < n && n <= 15) + { + // "0.[000]digits" (n <= 0) is the digits after 1 - n leading '0's + // with the point after the first; "digits[000].0" (n >= k) and + // "dig.its" put the point after n characters + const int pad = n <= 0 ? 1 - n : 0; + const char* const s = s0 - pad; + const int len = k + pad; + const int point = n + pad; + std::memcpy(first, s, 16); + std::memcpy(first + point + 1, s + point, 24); + first[point] = '.'; + return first + (point >= len ? point + 2 : len + 1); + } + + // d.igitse+XX, with at least two exponent digits (as append_exponent()) + std::memcpy(first, s0, 16); + std::memcpy(first + 2, s0 + 1, 16); + first[1] = '.'; + char* const end = first + (k == 1 ? 1 : k + 1); + const int e = n - 1; + const auto ea = e < 0 ? 0u - static_cast(e) : static_cast(e); // (unsigned: no signed overflow to assume) + const bool three = ea >= 100; + end[0] = 'e'; + end[1] = e < 0 ? '-' : '+'; + end[2] = static_cast('0' + (three ? ea / 100 : (ea / 10) % 10)); + end[3] = static_cast('0' + (three ? (ea / 10) % 10 : ea % 10)); + end[4] = static_cast('0' + (ea % 10)); + return end + (three ? 5 : 4); +} + +/*! +@brief the shortest decimal of a positive double (Zmij), as write_decimal() +writes it + +For a normal double, the shorter candidate has 15 or 16 digits: they are +converted at once (two halves of eight digits) and followed by the digit +after them, if there is one, without the multiplication and division by 10 +that counting the digits of one number would take. The fixed layouts move +the digits after the point by one byte. + +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_shortest(char* first, const zmij::shortest_decimal d) noexcept +{ + const std::uint64_t sig = d.integral; + if (JSON_HEDLEY_UNLIKELY(sig < 100000000000000u || sig >= 10000000000000000u)) + { + // (subnormals) + return d.has_digit ? write_decimal(first, (sig * 10) + d.digit, d.exponent) : write_decimal(first, sig, d.exponent + 1); + } + const bool sixteen = sig >= 1000000000000000u; // (else 15 digits) + const int last = d.has_digit ? d.digit : 0; + const std::uint64_t upper = sig / 100000000u; +#if JSON_DTOA_SSE2 + // NOLINTBEGIN(portability-simd-intrinsics) + // the two halves in the 64-bit lanes, each as abcd * 2^32 + efgh, then as + // bytes (as eight_digit_bytes(), one lane each) + const __m128i x = _mm_set_epi64x(static_cast(sig - (upper * 100000000u)), static_cast(upper)); // NOLINT(runtime/int) + const __m128i abcd = _mm_srli_epi64(_mm_mul_epu32(x, _mm_set1_epi64x(109951163)), 40); // 2^40 / 10000 + 1 + const __m128i abcd_efgh = _mm_add_epi64(x, _mm_mul_epu32(abcd, _mm_set1_epi64x(4294957296))); // 2^32 - 10000 + // 32-bit lanes in the order of the text: abcd, efgh of both halves + const __m128i fours = _mm_shuffle_epi32(abcd_efgh, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128i ab = _mm_srli_epi16(_mm_mulhi_epu16(fours, _mm_set1_epi32(5243)), 3); + const __m128i ab_cd = _mm_or_si128(_mm_slli_epi32(_mm_sub_epi16(fours, _mm_mullo_epi16(ab, _mm_set1_epi32(100))), 16), ab); + // 16-bit lanes ab (< 100) -> bytes a, b: 256 * ab - 2559 * (ab / 10) + const __m128i bytes = _mm_sub_epi16(_mm_slli_epi16(ab_cd, 8), _mm_mullo_epi16(_mm_set1_epi16(2559), _mm_mulhi_epu16(ab_cd, _mm_set1_epi16(6554)))); + // the last digit that is not 0 (sig is not 0) + const auto nonzero = static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(bytes, _mm_setzero_si128()))); + const int digits = 63 - count_leading_zeros(nonzero) + (sixteen ? 1 : 0); // without trailing zeros + const __m128i chars = _mm_add_epi8(bytes, _mm_set1_epi8('0')); + // the 16 characters from the first digit + const __m128i s = sixteen ? chars : _mm_or_si128(_mm_srli_si128(chars, 1), _mm_slli_si128(_mm_cvtsi32_si128('0' + last), 15)); + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [&s](char* p) noexcept + { + std::memcpy(p, &s, 16); + }; + const char first_digit = static_cast(_mm_cvtsi128_si32(s)); + // NOLINTEND(portability-simd-intrinsics) +#elif JSON_DTOA_NEON + // as with SSE2: the halves in 32-bit lanes, then abcd, efgh of both + const uint32x2_t halves = vcreate_u32(upper | ((sig - (upper * 100000000u)) << 32u)); + const uint32x2_t abcd = vmovn_u64(vshrq_n_u64(vmull_n_u32(halves, static_cast(((std::uint64_t{1} << 40u) / 10000u) + 1u)), 40)); + const uint32x2_t efgh = vmls_n_u32(halves, abcd, 10000u); + const uint32x4_t fours = vcombine_u32(vzip1_u32(abcd, efgh), vzip2_u32(abcd, efgh)); + const uint32x4_t ab = vshrq_n_u32(vmulq_n_u32(fours, 5243u), 19); + const uint16x8_t ab_cd = vreinterpretq_u16_u32(vorrq_u32(ab, vshlq_n_u32(vmlsq_n_u32(fours, ab, 100u), 16))); + const uint16x8_t tens = vshrq_n_u16(vmulq_n_u16(ab_cd, 103u), 10); + const uint8x16_t bytes = vreinterpretq_u8_u16(vorrq_u16(tens, vshlq_n_u16(vmlsq_n_u16(ab_cd, tens, 10u), 8))); + // the last digit that is not 0 (sig is not 0): a nibble per byte + const std::uint64_t nonzero = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vtstq_u8(bytes, bytes)), 4)), 0); + const int digits = ((63 - count_leading_zeros(nonzero)) / 4) + (sixteen ? 1 : 0); // without trailing zeros + const uint8x16_t chars = vaddq_u8(bytes, vdupq_n_u8('0')); + // the 16 characters from the first digit + const uint8x16_t s = sixteen ? chars : vextq_u8(chars, vdupq_n_u8(static_cast('0' + last)), 1); + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [&s](char* p) noexcept + { + vst1q_u8(reinterpret_cast(p), s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + }; + const auto first_digit = static_cast(vgetq_lane_u8(s, 0)); +#else + const std::uint64_t hi = eight_digit_bytes(upper); + const std::uint64_t lo = eight_digit_bytes(sig - (upper * 100000000u)); + // trailing zero digits: zero bytes (sig is not 0) + const int zeros = lo != 0 ? count_trailing_zeros(lo) / 8 : 8 + (count_trailing_zeros(hi) / 8); + const int digits = 15 - zeros + (sixteen ? 1 : 0); // without trailing zeros + // the 16 characters from the first digit + const std::uint64_t s_hi = (sixteen ? hi : (hi << 8u) | (lo >> 56u)) + 0x3030303030303030u; + const std::uint64_t s_lo = (sixteen ? lo : (lo << 8u) | static_cast(last)) + 0x3030303030303030u; + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [s_hi, s_lo](char* p) noexcept + { + store_msb_first(p, s_hi); + store_msb_first(p + 8, s_lo); + }; + const auto first_digit = static_cast(s_hi >> 56u); +#endif + const int len = d.has_digit ? 16 + (sixteen ? 1 : 0) : digits; // significant digits + const int n = 16 + (sixteen ? 1 : 0) + d.exponent; // digits before the point + + if (JSON_HEDLEY_LIKELY(n >= 1 && n <= 15)) + { + // "dig.its" and "digits[000].0": the digits after the point move by + // one byte ('0's follow the digits) +#if JSON_DTOA_SSE2 + // NOLINTBEGIN(portability-simd-intrinsics) + // (in the register: reading the digits back from memory right after + // storing them waits until the stores are done) + const __m128i at = _mm_set1_epi8(static_cast(n)); + const __m128i index = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const __m128i before = _mm_cmpgt_epi8(at, index); + const __m128i after = _mm_cmpgt_epi8(index, at); + const __m128i text = _mm_or_si128(_mm_or_si128(_mm_and_si128(s, before), _mm_and_si128(_mm_slli_si128(s, 1), after)), + _mm_andnot_si128(_mm_or_si128(before, after), _mm_set1_epi8('.'))); + std::memcpy(first, &text, 16); + first[16] = static_cast(_mm_extract_epi16(s, 7) >> 8); + first[17] = s16; + // NOLINTEND(portability-simd-intrinsics) +#elif JSON_DTOA_NEON + const uint8x16_t index = vcombine_u8(vcreate_u8(0x0706050403020100u), vcreate_u8(0x0F0E0D0C0B0A0908u)); + const uint8x16_t at = vdupq_n_u8(static_cast(n)); + const uint8x16_t after_point = vbslq_u8(vcgtq_u8(index, at), vextq_u8(vdupq_n_u8(0), s, 15), vdupq_n_u8('.')); + vst1q_u8(reinterpret_cast(first), vbslq_u8(vcltq_u8(index, at), s, after_point)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + first[16] = static_cast(vgetq_lane_u8(s, 15)); + first[17] = s16; +#else + store_16(first); + first[16] = s16; + std::uint64_t after_point[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(after_point, first + n, 16); + std::memcpy(first + n + 1, after_point, 16); + first[n] = '.'; +#endif + return first + (n >= len ? n + 2 : len + 1); + } + if (n <= 0 && n > -4) + { + // "0.[000]digits" + std::memset(first, '0', 8); + first[1] = '.'; + store_16(first + 2 - n); + first[18 - n] = s16; + return first + 2 - n + len; + } + // d.igitse+XX, with at least two exponent digits (as append_exponent()) + store_16(first + 1); + first[17] = s16; + first[0] = first_digit; + first[1] = '.'; + char* const end = first + (len == 1 ? 1 : len + 1); + const int e = n - 1; + const auto ea = e < 0 ? 0u - static_cast(e) : static_cast(e); // (unsigned: no signed overflow to assume) + const bool three = ea >= 100; + end[0] = 'e'; + end[1] = e < 0 ? '-' : '+'; + end[2] = static_cast('0' + (three ? ea / 100 : (ea / 10) % 10)); + end[3] = static_cast('0' + (three ? (ea / 10) % 10 : ea % 10)); + end[4] = static_cast('0' + (ea % 10)); + return end + (three ? 5 : 4); +} + +/*! +@brief whether FloatType is an IEEE 754 binary64 type (a double, or a long double +that has the same format, as with MSVC and on Apple's Arm CPUs) + +These are the types the conversion of Zmij (see zmij.hpp) is used for; all +others (binary32, or a format the library does not know) use Grisu2. +*/ +template +constexpr bool has_binary64_format() noexcept +{ + return std::numeric_limits::is_iec559 + && std::numeric_limits::digits == 53 + && std::numeric_limits::max_exponent == 1024 + && sizeof(FloatType) == sizeof(std::uint64_t); +} + +template +struct is_binary64 : std::integral_constant()> {}; + +/// a positive finite float (other than binary64): Grisu2 and format_buffer() +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive_grisu2(char* first, const char* last, FloatType value) +{ + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); + static_cast(last); // (only used in the assertion) + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e., the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + grisu2(first, len, decimal_exponent, value); + + JSON_ASSERT(len <= std::numeric_limits::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits::digits10; + + JSON_ASSERT(last - first >= kMaxExp + 2); + JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); + + return format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +/// a positive finite binary64 number: the shortest digits (Zmij), laid out by +/// write_shortest() (through a local buffer if [first, last) is shorter than +/// the 41 bytes it may write) +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive_zmij(char* first, const char* last, FloatType value) +{ + static_assert(is_binary64::value, + "internal error: the conversion of Zmij needs IEEE 754 binary64 numbers"); + std::uint64_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + const zmij::shortest_decimal d = zmij::to_shortest(bits); + if (JSON_HEDLEY_LIKELY(last - first >= 41)) + { + return write_shortest(first, d); + } + std::array buf; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + const auto len = static_cast(write_shortest(buf.data(), d) - buf.data()); + JSON_ASSERT(last - first >= static_cast(len)); + std::memcpy(first, buf.data(), len); + return first + len; +} + +/// a positive finite binary64 number: Zmij (as a long double has the format of +/// a double here, its bits are those of the double of the same value) +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value, std::true_type /*is_binary64*/) +{ + return write_positive_zmij(first, last, value); +} + +/// a positive finite float of any other format: Grisu2 +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value, std::false_type /*is_binary64*/) +{ + return write_positive_grisu2(first, last, value); +} + +/// a positive finite float: Zmij for binary64 numbers, Grisu2 otherwise +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value) +{ + return write_positive(first, last, value, is_binary64 {}); +} + } // namespace dtoa_impl /*! @@ -1064,7 +1461,6 @@ JSON_HEDLEY_NON_NULL(1, 2) JSON_HEDLEY_RETURNS_NON_NULL char* to_chars(char* first, const char* last, FloatType value) { - static_cast(last); // maybe unused - fix warning JSON_ASSERT(std::isfinite(value)); // Use signbit(value) instead of (value < 0) since signbit works for -0. @@ -1090,28 +1486,7 @@ char* to_chars(char* first, const char* last, FloatType value) JSON_HEDLEY_DIAGNOSTIC_POP #endif - JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); - - // Compute v = buffer * 10^decimal_exponent. - // The decimal digits are stored in the buffer, which needs to be interpreted - // as an unsigned decimal integer. - // len is the length of the buffer, i.e., the number of decimal digits. - int len = 0; - int decimal_exponent = 0; - dtoa_impl::grisu2(first, len, decimal_exponent, value); - - JSON_ASSERT(len <= std::numeric_limits::max_digits10); - - // Format the buffer like printf("%.*g", prec, value) - constexpr int kMinExp = -4; - // Use digits10 here to increase compatibility with version 2. - constexpr int kMaxExp = std::numeric_limits::digits10; - - JSON_ASSERT(last - first >= kMaxExp + 2); - JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); - JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); - - return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); + return dtoa_impl::write_positive(first, last, value); } } // namespace detail diff --git a/include/nlohmann/detail/conversions/to_json.hpp b/include/nlohmann/detail/conversions/to_json.hpp index d0d9610f4..5b217d7fe 100644 --- a/include/nlohmann/detail/conversions/to_json.hpp +++ b/include/nlohmann/detail/conversions/to_json.hpp @@ -167,6 +167,17 @@ struct external_constructor j.replace_value(value_t::array, value); } + /// an rvalue container: move its elements + template + static void construct_moving(BasicJsonType& j, CompatibleArrayType& arr) + { + using std::begin; + using std::end; + + const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(arr)), std::make_move_iterator(end(arr)))); + j.replace_value(value_t::array, value); + } + template static void construct(BasicJsonType& j, const std::vector& arr) { @@ -217,6 +228,17 @@ struct external_constructor j.replace_value(value_t::object, value); } + /// an rvalue container: move its values + template + static void construct_moving(BasicJsonType& j, CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(obj)), std::make_move_iterator(end(obj)))); + j.replace_value(value_t::object, value); + } + template < typename BasicJsonType, typename CompatibleObjectType, enable_if_t < !std::is_same::value, int > = 0 > static void construct(BasicJsonType& j, const CompatibleObjectType& obj) @@ -303,14 +325,14 @@ inline void to_json(BasicJsonType& j, FloatType val) noexcept } template::value, int> = 0> + enable_if_t, CompatibleNumberUnsignedType>::value, int> = 0> inline void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept { external_constructor::construct(j, static_cast(val)); } template::value, int> = 0> + enable_if_t::value, int> = 0> inline void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept { external_constructor::construct(j, static_cast(val)); @@ -354,6 +376,66 @@ inline void to_json(BasicJsonType& j, const CompatibleArrayType& arr) external_constructor::construct(j, arr); } +// types that have a dedicated to_json overload and must not be treated as generic containers +template +struct has_dedicated_to_json : std::false_type {}; + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template<> +struct has_dedicated_to_json : std::true_type {}; +#endif + +#if defined(__cpp_lib_char8_t) +template +struct has_dedicated_to_json> : std::true_type {}; +#endif + +#if JSON_HAS_RANGE_VIEW_CONVERSION +template +struct is_range_view_type : is_compatible_range_view {}; +#else +template +struct is_range_view_type : std::false_type {}; +#endif + +#if JSON_USE_OBJECTS_FOR_ENUM_KEYED_MAPS +template +struct is_enum_keyed_map_type : is_enum_keyed_map {}; +#else +template +struct is_enum_keyed_map_type : std::false_type {}; +#endif + +// rvalue (non-const) container: the generic to_json takes its elements by move +template +using is_movable_container_rvalue = negation, + std::is_const, + has_dedicated_to_json, + is_basic_json, + is_range_view_type, + is_enum_keyed_map_type, + std::is_same>>>; + +// rvalue array-like containers +template +using is_rvalue_array_to_json = conjunction, + is_compatible_array_type, + negation, + std::is_same, + is_compatible_object_type, + is_compatible_string_type, + is_compatible_binary_type>>>; + +// rvalue containers: their elements are moved instead of copied +template::value, int> = 0> +inline void to_json(BasicJsonType& j, CompatibleArrayType && arr) +{ + external_constructor::construct_moving(j, arr); +} + #if JSON_HAS_RANGE_VIEW_CONVERSION template < typename BasicJsonType, typename T, enable_if_t < is_compatible_range_view>::value @@ -427,6 +509,19 @@ inline void to_json(BasicJsonType& j, const EnumKeyedMap& map) } #endif +// rvalue object-like containers +template +using is_rvalue_object_to_json = conjunction, + negation>, + is_compatible_object_type>; + +template::value, int> = 0> +inline void to_json(BasicJsonType& j, CompatibleObjectType && obj) +{ + external_constructor::construct_moving(j, obj); +} + template inline void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) { diff --git a/include/nlohmann/detail/conversions/zmij.hpp b/include/nlohmann/detail/conversions/zmij.hpp new file mode 100644 index 000000000..3820d891a --- /dev/null +++ b/include/nlohmann/detail/conversions/zmij.hpp @@ -0,0 +1,222 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2025 Victor Zverovich +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // array +#include // size_t +#include // uint32_t, uint64_t + +#include +#include +#include +#include + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief the shortest decimal representation of a double + +A C++11 port of the conversion of Zmij by Victor Zverovich +(https://github.com/vitaut/zmij, MIT license): the shortest decimal in the +rounding interval of a double, the closest one if there are several. Zmij +credits Xiang JunBo (producing the shorter candidate without a division) and +Dougall Johnson (the compressed powers of ten). The powers of ten are taken +from the table for number parsing (pow5_table.hpp) where it holds them, and +computed from the compressed tables of Zmij beyond it. +*/ +namespace zmij +{ + +/// the compressed powers of ten of Zmij +inline const std::array& pow10_minor() noexcept +{ + static const std::array table = + { + { + 0x8000000000000000u, 0xa000000000000000u, 0xc800000000000000u, 0xfa00000000000000u, 0x9c40000000000000u, + 0xc350000000000000u, 0xf424000000000000u, 0x9896800000000000u, 0xbebc200000000000u, 0xee6b280000000000u, + 0x9502f90000000000u, 0xba43b74000000000u, 0xe8d4a51000000000u, 0x9184e72a00000000u, 0xb5e620f480000000u, + 0xe35fa931a0000000u, 0x8e1bc9bf04000000u, 0xb1a2bc2ec5000000u, 0xde0b6b3a76400000u, 0x8ac7230489e80000u, + 0xad78ebc5ac620000u, 0xd8d726b7177a8000u, 0x878678326eac9000u, 0xa968163f0a57b400u, 0xd3c21bcecceda100u, + 0x84595161401484a0u, 0xa56fa5b99019a5c8u, 0xcecb8f27f4200f3au + } + }; + return table; +} + +/// (high, low) pairs +inline const std::array& pow10_major() noexcept +{ + static const std::array table = + { + { + 0xaddcb9e83c6b1793u, 0xdf4abe242a1bbf3eu, 0xaf8e5410288e1b6fu, 0x07ecf0ae5ee44ddau, 0xb1442798f49ffb4au, 0x99cd11cfdf41779du, + 0xb2fe3f0b8599ef07u, 0x861fa7e6dcb4aa15u, 0xb4bca50b065abe63u, 0x0fed077a756b53aau, 0xb67f6455292cbf08u, 0x1a3bc84c17b1d543u, + 0xb84687c269ef3bfbu, 0x3d5d514f40eea742u, 0xba121a4650e4ddebu, 0x92f34d62616ce413u, 0xbbe226efb628afeau, 0x890489f70a55368cu, + 0xbdb6b8e905cb600fu, 0x5400e987bbc1c921u, 0xbf8fdb78849a5f96u, 0xde98520472bdd034u, 0xc16d9a0095928a27u, 0x75b7053c0f178294u, + 0xc350000000000000u, 0x0000000000000000u, 0xc5371912364ce305u, 0x6c28000000000000u, 0xc722f0ef9d80aad6u, 0x424d3ad2b7b97ef6u, + 0xc913936dd571c84cu, 0x03bc3a19cd1e38eau, 0xcb090c8001ab551cu, 0x5cadf5bfd3072cc6u, 0xcd036837130890a1u, 0x36dba887c37a8c10u, + 0xcf02b2c21207ef2eu, 0x94f967e45e03f4bcu, 0xd106f86e69d785c7u, 0xe13336d701beba52u, 0xd31045a8341ca07cu, 0x1ede48111209a051u, + 0xd51ea6fa85785631u, 0x552a74227f3ea566u, 0xd732290fbacaf133u, 0xa97c177947ad4096u, 0xd94ad8b1c7380874u, 0x18375281ae7822bdu, + 0xdb68c2ca82ed2a05u, 0xa67398db9f6820e1u + } + }; + return table; +} + +/// one bit per power: whether the computed value is one unit too large +inline const std::array& pow10_fixups() noexcept +{ + static const std::array table = + { + { + 0x8d8fc810u, 0x06100293u, 0x19000000u, 0x00100000u, 0x00000908u, 0x00000000u, 0x04e00300u, 0x3807e0b2u, 0x3d83d793u, 0x0006f5ccu, + 0x00000000u, 0xffff0000u, 0x8076337du, 0x4ff45ba0u, 0x09405033u, 0x034376d9u, 0x09000000u, 0x4e100501u, 0x076d14dcu, 0xf964f45eu, + 0x0000003du + } + }; + return table; +} + +/// the 128-bit significand of 10^k, rounded down, for k in [-307, 341] +/// (compute_pow10 of Zmij) +inline uint128_parts compute_pow10(int k) noexcept +{ + const auto i = static_cast(k + 307); + const std::uint64_t m = pow10_minor()[(i + 24) % 28]; + const std::size_t j = 2 * static_cast((i + 24) / 28); + const std::uint64_t h_hi = pow10_major()[j]; + const std::uint64_t h_lo = pow10_major()[j + 1]; + const std::uint64_t h1 = full_multiplication(h_lo, m).high; + const std::uint64_t c0 = h_lo * m; + const std::uint64_t c1 = h1 + (h_hi * m); + const std::uint64_t c2 = (c1 < h1 ? 1u : 0u) + full_multiplication(h_hi, m).high; + uint128_parts r{}; + if ((c2 >> 63u) != 0) + { + r.high = c2; + r.low = c1; + } + else + { + r.high = (c2 << 1u) | (c1 >> 63u); + r.low = (c1 << 1u) | (c0 >> 63u); + } + r.low -= (pow10_fixups()[i >> 5u] >> (i & 31u)) & 1u; + return r; +} + +/// The 128-bit significand of 10^k, rounded down, for k in [-342, 341]. +/// Up to 10^308, the table for number parsing holds the same significands +/// (those of 5^k), except for k in [-27, -1], where it holds them one unit +/// larger (as the Eisel-Lemire algorithm needs them). +inline uint128_parts pow10(int k) noexcept +{ + if (k > pow5_128_largest_power) + { + return compute_pow10(k); // (only for the smallest doubles) + } + const auto i = 2 * static_cast(k - pow5_128_smallest_power); + uint128_parts r{pow5_128()[i + 1], pow5_128()[i]}; + const std::uint64_t adjust = static_cast(k + 27) < 27u ? 1u : 0u; + r.high -= r.low < adjust ? 1u : 0u; + r.low -= adjust; + return r; +} + +/// (x_hi * 2^64 + x_lo) * y >> 64, as 128 bits +inline uint128_parts umul192_hi128(std::uint64_t x_hi, std::uint64_t x_lo, std::uint64_t y) noexcept +{ + const uint128_parts p = full_multiplication(x_hi, y); + uint128_parts r{}; + r.low = p.low + full_multiplication(x_lo, y).high; + r.high = p.high + (r.low < p.low ? 1u : 0u); + return r; +} + +/// (x * y + c) >> 64 +inline std::uint64_t umul128_add_hi64(std::uint64_t x, std::uint64_t y, std::uint64_t c) noexcept +{ + const uint128_parts p = full_multiplication(x, y); + return p.high + (p.low + c < p.low ? 1u : 0u); +} + +/// the result of Zmij: the shorter candidate and, if that is outside the +/// rounding interval, the digit after it (16 bytes: returned in registers) +struct shortest_decimal +{ + std::uint64_t integral; ///< the shorter candidate (15 or 16 digits for normal doubles) + int exponent; ///< the decimal exponent of the digit after it + unsigned char digit; ///< the digit after it (if has_digit) + bool has_digit; ///< whether the shortest decimal is integral * 10 + digit +}; + +/// The shortest decimal in the rounding interval of a positive finite double +/// given by its bits, the closest one if there are several (to_decimal of +/// Zmij, which keeps the last digit apart: the 15 or 16 digits before it can be +/// converted without a multiplication by 10 first). Always inlined: GCC +/// otherwise calls it, and its result goes through memory. +JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexcept +{ + constexpr int extra_shift = 9; + const auto raw_exp = static_cast((bits >> 52u) & 0x7FFu); + std::uint64_t bin_sig = bits & ((std::uint64_t{1} << 52u) - 1); + // a power of two has a narrower interval below (except the smallest normal) + const bool regular = bin_sig != 0 || raw_exp <= 1; + const int bin_exp = (raw_exp == 0 ? 1 : raw_exp) - 1075; + if (raw_exp != 0) + { + bin_sig |= std::uint64_t{1} << 52u; + } + // floor(log10(2^bin_exp)), or floor(log10(3/4 * 2^bin_exp)) for the irregular case + const int dec_exp = ((bin_exp * 315653) - (regular ? 0 : 131072)) >> 20; + // scaled by 10^(-dec_exp - 1): the integral part is the shorter candidate + const int shift = bin_exp + ((-(dec_exp + 1) * 217707) >> 16) + 1 + extra_shift; + const uint128_parts p10 = pow10(-dec_exp - 1); + const uint128_parts p = umul192_hi128(p10.high, p10.low, bin_sig << static_cast(shift)); + std::uint64_t integral = p.high >> static_cast(extra_shift); + // (no casts in the shifts: MSVC 2015 warns C4554 on them despite the parentheses) + constexpr unsigned fractional_low_shift = static_cast(extra_shift); + constexpr unsigned fractional_high_shift = 64u - fractional_low_shift; + const std::uint64_t fractional = (p.high << fractional_high_shift) | (p.low >> fractional_low_shift); + std::uint64_t digit = 0; + bool round_up = false; + bool round_down = false; + if (JSON_HEDLEY_LIKELY(regular)) + { + const std::uint64_t half_ulp = (p10.high >> static_cast(extra_shift + 1 - shift)) + (1 - (bin_sig & 1u)); + round_up = fractional + half_ulp < fractional; + round_down = half_ulp > fractional; + // the last digit of the longer candidate, rounded to nearest + digit = umul128_add_hi64(fractional, 10, (std::uint64_t{1} << 63u) + 6); + if (fractional == (std::uint64_t{1} << 62u)) + { + digit = 2; // 2.5 rounds to 2 + } + } + else + { + const std::uint64_t half_ulp = p10.high >> static_cast(extra_shift + 1 - shift); + round_up = half_ulp > ~std::uint64_t{0} - fractional; + round_down = (half_ulp >> 1u) > fractional; + digit = umul128_add_hi64(fractional, 10, (std::uint64_t{1} << 63u) - 1); + const std::uint64_t lowest = umul128_add_hi64(fractional - (half_ulp >> 1u), 10, ~std::uint64_t{0}); + digit = digit < lowest ? lowest : digit; + } + integral += round_up ? 1u : 0u; + // if the shorter candidate is outside the rounding interval: one digit more + return shortest_decimal{integral, dec_exp, static_cast(digit), !round_up && !round_down}; +} + +} // namespace zmij +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index f2c6651c4..b2b5f36dd 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -428,7 +428,7 @@ void templated_json_throw(ExceptionType exception) @throw type_error.302 always */ template -JSON_HEDLEY_NO_RETURN inline void throw_type_must_be(const char* expected, const BasicJsonType& j) +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_type_must_be(const char* expected, const BasicJsonType& j) { static_cast(expected); // unused when JSON_NOEXCEPTION is defined static_cast(j); @@ -443,7 +443,7 @@ JSON_HEDLEY_NO_RETURN inline void throw_type_must_be(const char* expected, const @throw type_error always */ template -JSON_HEDLEY_NO_RETURN inline void throw_cannot_use_with(const exception_id id_, const char* operation, const BasicJsonType& j) +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_cannot_use_with(const exception_id id_, const char* operation, const BasicJsonType& j) { static_cast(id_); // unused when JSON_NOEXCEPTION is defined static_cast(operation); @@ -451,6 +451,77 @@ JSON_HEDLEY_NO_RETURN inline void throw_cannot_use_with(const exception_id id_, JSON_THROW(type_error::create(id_, concat("cannot use ", operation, " with ", j.type_name()), &j)); } +/*! +@brief throws ExceptionType with a fixed message; kept out of line so that callers do not carry the exception construction +*/ +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_literal(const exception_id id_, const char* what_arg, BasicJsonContext context) +{ + static_cast(id_); // unused when JSON_NOEXCEPTION is defined + static_cast(what_arg); + static_cast(context); + JSON_THROW(ExceptionType::create(id_, what_arg, context)); +} + +// JSON Patch errors; constructed out of line so that patch_inplace() does not carry them + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_add_parent(const BasicJsonType& parent) +{ + static_cast(parent); + JSON_THROW(out_of_range::create(exception_id::patch_add_parent_not_container, concat("cannot add value: the JSON Patch 'add' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_remove_parent(const BasicJsonType& parent) +{ + static_cast(parent); + JSON_THROW(out_of_range::create(exception_id::patch_remove_parent_not_container, concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_not_an_array(const BasicJsonType& j) +{ + static_cast(j); + JSON_THROW(parse_error::create(exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &j)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_missing_member(const StringType& op, const StringType& member, const bool string_type, const BasicJsonType& val) +{ + static_cast(op); + static_cast(member); + static_cast(string_type); + static_cast(val); + // context-sensitive error message + const std::string error_msg = (op == "op") ? std::string("operation") : concat("operation '", op, '\''); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat(error_msg, string_type ? " must have string member '" : " must have member '", member, "'"), &val)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_move_into_child(const StringType& from_path, const StringType& path, const BasicJsonType& result) +{ + static_cast(from_path); + static_cast(path); + static_cast(result); + JSON_THROW(out_of_range::create(exception_id::patch_move_into_child, concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_test_failed(const BasicJsonType& val) +{ + static_cast(val); + JSON_THROW(other_error::create(exception_id::patch_test_failed, concat("unsuccessful: ", val.dump()), &val)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_invalid_operation(const StringType& op, const BasicJsonType& val) +{ + static_cast(op); + static_cast(val); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat("operation value '", op, "' is invalid"), &val)); +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 1612da026..8e53f197f 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -223,7 +223,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current != char_traits::eof())) { - return last_byte_error(exception_id::unexpected_end_of_input, concat("expected end of input; last byte: 0x", get_token_string()), "value"); + return last_byte_error(exception_id::unexpected_end_of_input, "expected end of input", "value", ""); } } @@ -354,7 +354,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) { - static_cast(last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document")); + static_cast(bson_document_size_error(document_size, chars_read - document_start)); return repair_requested(); } return true; @@ -678,7 +678,7 @@ class binary_reader { // the end of the element is lost (see resync) skip_requested = true; - return last_byte_error(exception_id::unexpected_byte, concat("string length must be at least 1, is ", std::to_string(len)), "string"); + return bson_length_error("string length must be at least 1, is ", len, "string"); } if (JSON_HEDLEY_UNLIKELY(!get_string(len - static_cast(1), result))) @@ -713,7 +713,7 @@ class binary_reader { // the end of the element is lost (see resync) skip_requested = true; - return last_byte_error(exception_id::unexpected_byte, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"); + return bson_length_error("byte array length cannot be negative, is ", len, "binary"); } // All BSON binary values have a subtype @@ -803,9 +803,7 @@ class binary_reader default: // anything else is not supported (yet) { - const std::string cr_str = hex_byte(static_cast(element_type)); - static_cast(report_error(element_type_parse_position, cr_str, - parse_error::create(exception_id::bson_unsupported_type, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr))); + static_cast(bson_unsupported_type_error(element_type_parse_position, element_type)); return skip_unsupported_bson_element(element_type); } } @@ -1259,6 +1257,7 @@ class binary_reader @param[in] context parsing context for the error message @return false, so that the caller stops reading */ + JSON_HEDLEY_NEVER_INLINE bool cbor_indefinite_string_error(const char* type_name, const char* context) { return last_byte_error(exception_id::invalid_string_or_size, concat("indefinite-length ", type_name, " is not allowed inside indefinite-length ", type_name, "; last byte: 0x", get_token_string()), context); @@ -1338,7 +1337,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x60-0x7B)", inside_indefinite ? "" : " or indefinite string type (0x7F)", "; last byte: 0x", get_token_string()), "string"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x60-0x7B)", "string", inside_indefinite ? "" : " or indefinite string type (0x7F)"); } } } @@ -1472,7 +1471,7 @@ class binary_reader // a key that begins an item is skipped with its value when recovering // (see resync); a break stop code or a reserved byte begins none skip_requested = is_cbor_item_head(static_cast(current)); - return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key"); + return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found); } /*! @@ -1553,7 +1552,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x40-0x5B)", inside_indefinite ? "" : " or indefinite binary array type (0x5F)", "; last byte: 0x", get_token_string()), "binary"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x40-0x5B)", "binary", inside_indefinite ? "" : " or indefinite binary array type (0x5F)"); } } } @@ -1699,8 +1698,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message(concat("excessive ", context, " size"), "size"), nullptr)); + return container_size_error(context); } result = conditional_static_cast(len); return true; @@ -2388,7 +2386,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", get_token_string()), "string"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0xA0-0xBF, 0xD9-0xDB)", "string", ""); } } } @@ -2483,7 +2481,7 @@ class binary_reader // the key is skipped with its value when recovering (see resync) skip_requested = true; - return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key"); + return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found); } /*! @@ -3128,8 +3126,7 @@ class binary_reader } if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(number))) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("integer value overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char return true; @@ -3226,8 +3223,7 @@ class binary_reader } if (!value_in_range_of(number)) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("integer value overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } result = detail::conditional_static_cast(number); return true; @@ -3308,7 +3304,7 @@ class binary_reader // or SIZE_MAX. if (JSON_HEDLEY_UNLIKELY(result > (std::numeric_limits::max)() / i)) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size"); } result *= i; // the pre-check above already rules out result becoming 0 @@ -3317,7 +3313,7 @@ class binary_reader // unknown-size container (see get_ubjson_size_type()) if (result == npos) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size"); } if (JSON_HEDLEY_UNLIKELY(!emit_unsigned(i))) { @@ -3368,7 +3364,7 @@ class binary_reader if (input_format == input_format_t::bjdata && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) { - return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type"); + return optimized_type_error(); } if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) @@ -3383,7 +3379,7 @@ class binary_reader { return false; } - return last_byte_error(exception_id::unexpected_byte, concat("expected '#' after type information; last byte: 0x", get_token_string()), "size"); + return last_byte_error(exception_id::unexpected_byte, "expected '#' after type information", "size", ""); } const bool is_error = get_ubjson_size_value(result.first, is_ndarray, 0, result.second); @@ -3534,7 +3530,7 @@ class binary_reader } if (JSON_HEDLEY_UNLIKELY(current > 127)) { - static_cast(last_byte_error(exception_id::invalid_string_or_size, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", get_token_string()), "char")); + static_cast(last_byte_error(exception_id::invalid_string_or_size, "byte after 'C' must be in range 0x00..0x7F", "char", "")); if (!repair_requested()) { return false; @@ -3628,8 +3624,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') && size_and_type.first > max_valueless_container_size)) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("excessive array size", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive array size", "size"); } if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) @@ -4540,7 +4535,8 @@ class binary_reader { // in case of failure, advance position by 1 to report the failing location ++chars_read; - return report_error(chars_read, "", parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + static_cast(report_eof(context)); + return false; } return true; } @@ -4692,8 +4688,7 @@ class binary_reader const auto result = static_cast(number); if (JSON_HEDLEY_UNLIKELY(std::isfinite(number) && !std::isfinite(result))) { - static_cast(report_error(chars_read, get_token_string(), - out_of_range::create(exception_id::number_overflow, exception_message("number overflow", "value"), nullptr))); + static_cast(out_of_range_error(exception_id::number_overflow, "number overflow", "value")); // when recovering, the number becomes infinity, as in JSON text if (!repair_requested()) { @@ -5213,22 +5208,82 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(current == char_traits::eof())) { - return report_error(chars_read, "", - parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + return report_eof(context); } return true; } + /*! + @brief reports an unexpected end of input (out of line to keep the exception + construction out of the hot reading functions) + @param[in] context further context information (for diagnostics) + @return false, so that the caller stops reading (see report_error) + */ + JSON_HEDLEY_NON_NULL(2) + JSON_HEDLEY_NEVER_INLINE + bool report_eof(const char* context) + { + return report_error(chars_read, "", + parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + } + + /*! + @brief reports an out_of_range error at the last read byte (out of line) + */ + JSON_HEDLEY_NEVER_INLINE + bool out_of_range_error(const exception_id id_, const char* detail, const char* context) + { + return report_error(chars_read, get_token_string(), out_of_range::create(id_, exception_message(detail, context), nullptr)); + } + + JSON_HEDLEY_NEVER_INLINE + bool container_size_error(const char* what) + { + return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, + exception_message(concat("excessive ", what, " size"), "size"), nullptr)); + } + + JSON_HEDLEY_NEVER_INLINE + bool optimized_type_error() + { + return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type"); + } + + template + JSON_HEDLEY_NEVER_INLINE + bool bson_length_error(const char* prefix, const NumberType len, const char* context) + { + return last_byte_error(exception_id::unexpected_byte, concat(prefix, std::to_string(len)), context); + } + + JSON_HEDLEY_NEVER_INLINE + bool bson_document_size_error(const std::int32_t document_size, const std::size_t bytes_read) + { + return last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(bytes_read), ")"), "document"); + } + + JSON_HEDLEY_NEVER_INLINE + bool bson_unsupported_type_error(const std::size_t position, const char element_type) + { + const std::string cr_str = hex_byte(static_cast(element_type)); + return report_error(position, cr_str, + parse_error::create(exception_id::bson_unsupported_type, position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + } + /*! @brief reports a parse error at the last read byte @param[in] id_ the id of the parse_error exception @param[in] detail a detailed error message @param[in] context further context information + @param[in] tail optional text appended to the detail, followed by "; last byte: 0x" @return false, so that the caller stops reading (see report_error) */ - bool last_byte_error(const exception_id id_, const std::string& detail, const std::string& context) + JSON_HEDLEY_NEVER_INLINE + bool last_byte_error(const exception_id id_, const std::string& detail, const char* context, const char* tail = nullptr) { - return report_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, exception_message(detail, context), nullptr)); + // with a tail, the message reads "; last byte: 0x" + return report_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, + exception_message(tail == nullptr ? detail : concat(detail, tail, "; last byte: 0x", get_token_string()), context), nullptr)); } /*! @@ -5237,7 +5292,8 @@ class binary_reader @param[in] context further context information @return false, so that the caller stops reading (see report_error) */ - bool unexpected_byte(const char* detail, const std::string& context) + JSON_HEDLEY_NEVER_INLINE + bool unexpected_byte(const char* detail, const char* context) { return last_byte_error(exception_id::unexpected_byte, concat(detail, ": 0x", get_token_string()), context); } @@ -5248,6 +5304,7 @@ class binary_reader @param[in] context further context information @return false, so that the caller stops reading (see report_error) */ + JSON_HEDLEY_NEVER_INLINE bool length_type_error(const char* position, const char* context) { const char* types = input_format == input_format_t::bjdata ? "U, i, u, I, m, l, M, L" : "U, i, I, l, L"; diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 847890f7a..2df78dabb 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -8,12 +8,12 @@ #pragma once -#include // min +#include // count, find, min #include // array #include // size_t #include // uint32_t #include // strlen -#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer @@ -297,6 +297,7 @@ class iterator_input_adapter return result; } + ++eof_reads; return char_traits::eof(); } @@ -306,6 +307,26 @@ class iterator_input_adapter return static_cast(std::distance(begin, current)); } + // number of get_character() calls so far, including those that returned + // EOF; the lexer derives its position from it (see lexer::lazy_position) + std::size_t get_read_count() const + { + return get_consumed_count() + eof_reads; + } + + // number of newlines in the consumed range [0, last_index); @a line_start + // is set to the index after the last of them (0 if there is none); only + // valid when supports_seek + std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const + { + const auto to = std::next(begin, static_cast::difference_type>(last_index)); + const auto newline = char_traits::to_char_type('\n'); + using reverse_it = std::reverse_iterator; + const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline); + line_start = static_cast(std::distance(begin, last_newline.base())); + return static_cast(std::count(begin, last_newline.base(), newline)); + } + // append the already-consumed characters in the half-open range // [first_index, last_index) to @a out; only valid when supports_seek template @@ -422,6 +443,8 @@ class iterator_input_adapter IteratorType begin; IteratorType current; SentinelType end; + /// number of get_character() calls that returned EOF + std::size_t eof_reads = 0; template friend struct wide_string_input_helper; diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 3c71b28c0..f8e952ba6 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -189,12 +189,19 @@ Shared by json_sax_dom_parser and json_sax_dom_callback_parser. @param[in] ref the object or array that was just created @return whether parsing should continue (false after reporting out_of_range.408) */ +template +JSON_HEDLEY_NEVER_INLINE +bool container_too_large_error(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref) +{ + return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref)); +} + template bool check_container_size(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref) { if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref->max_size())) { - return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref)); + return container_too_large_error(sax, len, kind, ref); } return true; } @@ -219,7 +226,7 @@ struct diagnostic_positions { // Lexer has read the first character of the container, so // subtract 1 from the position to get the correct start position. - v.start_position = lexer->get_position() - 1; + v.start_position = lexer->get_chars_read_total() - 1; } } @@ -233,7 +240,7 @@ struct diagnostic_positions if (lexer) { // Lexer's position is past the closing brace or bracket, so set that as the end position. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); } } @@ -249,7 +256,7 @@ struct diagnostic_positions // Lexer has read past the current field value, so set the end position to the current position. // The start position will be set below based on the length of the string representation // of the value. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); switch (v.type()) { @@ -679,16 +686,19 @@ class json_sax_dom_callback_parser if (keep && ref_stack.back()) { auto& obj = *ref_stack.back()->m_data.m_value.object; - const auto it = obj.find(val); - if (it != obj.end()) + // a single lookup: insert the placeholder, or find the existing member + const auto inserted = obj.emplace(val, discarded); + const auto it = inserted.first; + if (!inserted.second) { // this is a duplicate key (legal in JSON); remember its // current value so it can be restored later if the new // value is rejected by the callback, instead of being // erased together with the discarded placeholder duplicate_key_stash.emplace_back(&(it->second), it->second); + it->second = discarded; } - object_element = &(obj[val] = discarded); + object_element = &(it->second); } return true; diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index a8d4d3ca2..dfceb0862 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -17,6 +17,7 @@ #include // move #include // vector +#include #include #include #include @@ -186,6 +187,15 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether the read position is derived from the input adapter on demand + /// instead of being counted on every scanned character. Seekable adapters + /// know how much input they consumed and can revisit it, so + /// chars_read_total is the adapter's read count (minus a pending unget), + /// and line and column are recomputed from the consumed input only when + /// get_position() is called (errors). Counting per character kept two + /// increments and a newline check in the hot path of every get(). + static constexpr bool lazy_position = lazy_token_string; + /// whether a simulated unget can be passed on to the input adapter, which /// then leaves the character in the input; see /// input_adapter_supports_lookahead @@ -195,7 +205,9 @@ class lexer : public lexer_base /// whether string scanning may bulk-consume runs of ordinary characters /// directly from a contiguous input buffer (SWAR fast path). This requires /// the token to be reconstructible lazily (lazy_token_string), so bypassing - /// the per-character capture in get() cannot lose error diagnostics. + /// the per-character capture in get() cannot lose error diagnostics. It + /// also implies lazy_position, so the bulk paths need not advance the + /// position counters. static constexpr bool bulk_scan = lazy_token_string && input_adapter_supports_bulk_scan(is_detected {}); @@ -221,6 +233,41 @@ class lexer : public lexer_base // scan functions ///////////////////// + /// contiguous input: try to decode the 4 hex digits following `\\u` + /// directly from the input buffer via hex_codepoint(), instead of 4 calls + /// to get(). On success, advances the adapter exactly as those 4 get() + /// calls would (bulk_scan implies lazy_position, so there are no position + /// counters to advance) and leaves @a current holding the last of + /// the 4 digits, just as the last such get() would; the codepoint is + /// written to @a out. Makes no state change and returns false for a + /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not + /// being a hex digit, so the caller falls back unchanged to the + /// per-character loop, which then reports the same diagnostic (stopping + /// at the first invalid digit) as before this optimization. + bool get_codepoint_bulk(std::true_type /*bulk*/, int& out) + { + if (next_unget || ia.bulk_remaining() < 4) + { + return false; + } + const char_type* const raw = ia.bulk_data(); + const int codepoint = hex_codepoint(reinterpret_cast(raw)); + if (codepoint < 0) + { + return false; + } + ia.bulk_skip(4); + current = char_traits::to_int_type(raw[3]); + out = codepoint; + return true; + } + + /// streaming input: no bulk fast path + bool get_codepoint_bulk(std::false_type /*bulk*/, int& /*out*/) const noexcept + { + return false; + } + /*! @brief get codepoint from 4 hex characters following `\\u` @@ -240,6 +287,14 @@ class lexer : public lexer_base { // this function only makes sense after reading `\u` JSON_ASSERT(current == 'u'); + + // contiguous input: decode all 4 hex digits directly from the buffer + int fast_codepoint = 0; + if (get_codepoint_bulk(std::integral_constant {}, fast_codepoint)) + { + return fast_codepoint; + } + int codepoint = 0; const auto factors = { 12u, 8u, 4u, 0u }; @@ -331,10 +386,6 @@ class lexer : public lexer_base } token_buffer.append(reinterpret_cast(data), pos); ia.bulk_skip(pos); - // the run contains no newline (all bytes < 0x20 are treated as special), - // so only the flat character counters advance - position.chars_read_total += pos; - position.chars_read_current_line += pos; } /// streaming input: no bulk fast path @@ -1058,9 +1109,11 @@ class lexer : public lexer_base token_type::parse_error otherwise @note The scanner is independent of the current locale: token_buffer - always holds `.`. Only the std::strtod fallback of convert_number() - depends on the locale, and it looks up the decimal point right - before converting (see detail::convert_float_locale_aware()). + always holds `.`. The conversion of float and double does not use + the locale either. Only the std::strtold fallback of + convert_number() for long double formats other than binary64 + depends on it, and it looks up the decimal point right before + converting (see detail::convert_float_locale_aware()). */ token_type scan_number() // lgtm [cpp/use-of-goto] `goto` is used in this function to implement the number-parsing state machine described above. By design, any finite input will eventually reach the "done" state or return token_type::parse_error. In each intermediate state, 1 byte of the input is appended to the token_buffer vector, and only the already initialized variables token_buffer, number_type, and error_message are manipulated. { @@ -1073,7 +1126,7 @@ class lexer : public lexer_base // offset just past the last mantissa byte in token_buffer (i.e. the // index of 'e'/'E', or the whole token when there is no exponent). - // convert_number() uses it to count significant digits; npos means + // convert_number() uses it to split the token; npos means // "not seen an exponent yet" and is resolved at scan_number_done std::size_t mantissa_end = std::string::npos; @@ -1403,8 +1456,8 @@ scan_number_done: @param[in] mantissa_end offset just past the last mantissa byte in token_buffer (the index of 'e'/'E', or token_buffer.size() when there is no exponent); - used to skip Clinger's fast path when it cannot - possibly succeed - see detail::mantissa_fits_clinger() + with decimal_point_position, it locates the parts + of a float token without scanning it again */ token_type convert_number(token_type number_type, std::size_t mantissa_end) { @@ -1458,10 +1511,11 @@ scan_number_done: } // this code is reached if we parse a floating-point number or if an - // integer conversion above overflowed. Prefer std::from_chars - // (Eisel-Lemire, locale-independent, correctly rounded) when available; - // otherwise the exact Clinger fast path (double only); otherwise the - // locale-aware strtof/strtod/strtold. + // integer conversion above overflowed. float and double (and long + // double where it is binary64) are converted by the library itself, + // correctly rounded and independent of the locale; other long double + // formats use std::from_chars when available, otherwise the + // locale-aware strtold. if (convert_float_fast(num_begin, num_end, decimal_point_position, mantissa_end, value_float)) { return token_type::value_float; @@ -1581,8 +1635,6 @@ scan_number_done: if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) { ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return integer_result; } // The value does not fit an integer, so this token converts as a @@ -1599,8 +1651,6 @@ scan_number_done: decimal_point_position = dot_index; ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return convert_number(number_type, mantissa_end); } @@ -1651,7 +1701,7 @@ scan_number_done: #if JSON_DIAGNOSTIC_POSITIONS // the first character of the token has already been read, hence the -1 - token_start_position = position.chars_read_total - 1; + token_start_position = get_chars_read_total() - 1; #endif note_token_start(std::integral_constant {}); @@ -1704,6 +1754,15 @@ scan_number_done: /// per-character position counters (line-count-on-'\n' bookkeeping is /// handled afterwards, in track_after_read(), once `current` is known) void advance_position() noexcept + { + advance_position_impl(std::integral_constant {}); + } + + /// seekable adapter: the position is derived from the adapter on demand + void advance_position_impl(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: count the character + void advance_position_impl(std::false_type /*lazy*/) noexcept { ++position.chars_read_total; ++position.chars_read_current_line; @@ -1717,7 +1776,16 @@ scan_number_done: // seekable adapters reconstruct the token lazily on error (see // get_token_string), so the eager per-character copy is skipped capture_char(std::integral_constant {}); + track_newline(std::integral_constant {}); + return current; + } + /// seekable adapter: lines are counted on demand in get_position() + void track_newline(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: start a new line after reading a newline + void track_newline(std::false_type /*lazy*/) noexcept + { if (current == '\n') { ++position.lines_read; @@ -1726,8 +1794,6 @@ scan_number_done: chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } - - return current; } /*! @@ -1771,12 +1837,22 @@ scan_number_done: We implement unget by setting variable next_unget to true. The input is not changed - we just simulate ungetting by modifying chars_read_total, chars_read_current_line, and token_string. The next call to get() will - behave as if the unget character is read again. + behave as if the unget character is read again. With lazy_position, the + pending next_unget is all get_chars_read_total() needs. */ void unget() { next_unget = true; + unget_position(std::integral_constant {}); + uncapture_char(std::integral_constant {}); + } + /// seekable adapter: get_chars_read_total() accounts for next_unget + void unget_position(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: step the counters back over the ungotten character + void unget_position(std::false_type /*lazy*/) noexcept + { --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read @@ -1799,8 +1875,6 @@ scan_number_done: { --position.chars_read_current_line; } - - uncapture_char(std::integral_constant {}); } /// adapter without lookahead: nothing to do (see release_lookahead) @@ -1871,10 +1945,21 @@ scan_number_done: // diagnostics ///////////////////// - /// return position of last read token - constexpr position_t get_position() const noexcept + /*! + @brief return the number of characters read so far + + Unlike get_position(), this is O(1) for all adapters, so it may be called + for every value (JSON_DIAGNOSTIC_POSITIONS). + */ + std::size_t get_chars_read_total() const noexcept { - return position; + return get_chars_read_total_impl(std::integral_constant {}); + } + + /// return position of last read token + position_t get_position() const + { + return get_position_impl(std::integral_constant {}); } /*! @@ -1911,6 +1996,43 @@ scan_number_done: } #endif + private: + /// seekable adapter: every get() either read from the adapter (including + /// reads at EOF, which get() counts as characters as well) or replayed an + /// ungotten character, so the count is the adapter's read count minus a + /// still pending unget + std::size_t get_chars_read_total_impl(std::true_type /*lazy*/) const noexcept + { + return ia.get_read_count() - (next_unget ? 1u : 0u); + } + + /// streaming adapter: the counter is maintained by get() and unget() + std::size_t get_chars_read_total_impl(std::false_type /*lazy*/) const noexcept + { + return position.chars_read_total; + } + + /// seekable adapter: count the lines in the consumed input; O(input read + /// so far), but only called on errors + position_t get_position_impl(std::true_type /*lazy*/) const + { + position_t result; + result.chars_read_total = get_chars_read_total(); + // reads at EOF count as characters but are not part of the input + const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count()); + std::size_t line_start = 0; + result.lines_read = ia.count_consumed_lines(consumed, line_start); + result.chars_read_current_line = result.chars_read_total - line_start; + return result; + } + + /// streaming adapter: the position is maintained by get() and unget() + position_t get_position_impl(std::false_type /*lazy*/) const noexcept + { + return position; + } + + public: /// seekable adapter: rebuild the last read token from the input on demand const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const { @@ -2018,15 +2140,57 @@ scan_number_done: // into one that reloads the pointer from memory every character do { + if (current == '\n') + { + skip_indentation(std::integral_constant {}); + } get_ignoring_pending_unget(); } while (current_is_whitespace()); } + /*! + @brief contiguous input: skip the space indentation that follows a newline + 8 bytes at a time + + Pretty-printed JSON indents every line with a run of spaces, which the loop + in skip_whitespace() would otherwise read one character at a time. The + spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any + other whitespace are left to that loop. Skipping bypasses get(), which is + fine because bulk_scan implies lazy_position: the position is derived from + the adapter, not counted per character. + */ + void skip_indentation(std::true_type /*bulk*/) + { + JSON_ASSERT(!next_unget); + const std::size_t remaining = ia.bulk_remaining(); + if (remaining < 8) + { + return; + } + const char* const data = reinterpret_cast(ia.bulk_data()); + std::size_t pos = 0; + for (; pos + 8 <= remaining; pos += 8) + { + // little-endian, so the first byte is the lowest + const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull; + if (non_space != 0) + { + // skip the spaces before the first other byte of this word + pos += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + } + ia.bulk_skip(pos); + } + + /// streaming input: no bulk fast path + void skip_indentation(std::false_type /*bulk*/) const noexcept {} + token_type scan() { // initially, skip the BOM - if (position.chars_read_total == 0 && !skip_bom()) + if (get_chars_read_total() == 0 && !skip_bom()) { error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; return token_type::parse_error; @@ -2053,7 +2217,7 @@ scan_number_done: token_type scan_expecting(token_type expected_type) { JSON_ASSERT(expected_type == token_type::name_separator || expected_type == token_type::value_separator); - JSON_ASSERT(position.chars_read_total > 0); + JSON_ASSERT(get_chars_read_total() > 0); const char_int_type expected_char = static_cast((expected_type == token_type::name_separator) ? ':' : ','); skip_whitespace(); if (JSON_HEDLEY_LIKELY(current == expected_char)) diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index 8971f7193..3baeb60b0 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -9,6 +9,7 @@ #pragma once +#include // any_of #include // array #include // FLT_EVAL_METHOD #include // localeconv @@ -18,6 +19,8 @@ #include // memcpy #include // numeric_limits #include // string +#include // conditional, integral_constant, is_signed, true_type, false_type +#include // move #include #include @@ -35,15 +38,32 @@ #endif // This file contains the value-conversion helpers used by the lexer to turn an -// already-validated number token into a value, without the locale/errno -// overhead of std::strtoull/std::strtod where possible. They are free functions -// so the lexer stays focused on scanning (see lexer::convert_number()) and so -// that other parsers of JSON text can convert tokens exactly like it does. +// already-validated number token into a value. Integers and binary32/binary64 +// floats (float, double, and long double where it is binary64) are converted +// by the library itself, without the locale/errno overhead of +// std::strtoull/std::strtod and correctly rounded; other long double formats +// use std::from_chars or std::strtold. They are free functions so the lexer +// stays focused on scanning (see lexer::convert_number()) and so that other +// parsers of JSON text can convert tokens exactly like it does. NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { +// whether an integer of a signed type is negative; the overload for other types +// avoids comparing an unsigned value with zero +template +constexpr bool is_negative_integer(NumberType x, std::true_type /*is_signed*/) noexcept +{ + return x < 0; +} + +template +constexpr bool is_negative_integer(NumberType /*unused*/, std::false_type /*is_signed*/) noexcept +{ + return false; +} + /*! @brief fast integer parser for an already-validated unsigned integer @@ -77,8 +97,10 @@ bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedT x = (x * 10u) + digit; } value = static_cast(x); - // reject values that do not round-trip into a narrower NumberUnsignedType - return static_cast(value) == x; + // reject values that do not round-trip into a narrower NumberUnsignedType, + // and values that wrap around to negative numbers if it is a signed type + // (number_unsigned_t may be the same type as number_integer_t, #2573) + return static_cast(value) == x && !is_negative_integer(value, std::is_signed()); } /*! @@ -115,233 +137,291 @@ bool parse_integer_signed(const char* first, const char* last, NumberIntegerType } /*! -@brief exact fast path for parsing a `double` (Clinger's algorithm) - -For the common case - at most 19 significant digits, a decimal exponent in -[-22, 22], and a significand below 2^53 - the value equals significand * -10^exp computed in IEEE-754 double arithmetic, which is exact under -round-to-nearest because both operands are exactly representable. This is the -same fast path used by fast_float/simdjson; the general cases are left to -std::strtod. The parser only activates for number_float_t == double; float and -long double keep the std::strtof/std::strtold paths (see the templated overload -below). - -@param[in] first pointer to the first character of the number -@param[in] last pointer past the last character -@param[out] out the parsed value on success -@return true if the value was parsed exactly; false to fall back to strtod +@brief parameters of the IEEE-754 binary32 and binary64 formats for the float + conversion (after fast_float's binary_format) */ -inline bool parse_float_fast(const char* first, const char* last, double& out) noexcept +template +struct ieee_binary_format; + +template<> +struct ieee_binary_format<24> // binary32 { -#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 - // Clinger's fast path is only exact when double operations are evaluated in - // true double precision. On platforms that keep intermediates in extended - // precision (e.g. the x87 FPU on 32-bit x86, where FLT_EVAL_METHOD == 2) the - // single significand * 10^scale step is double-rounded and can be 1 ULP off, - // so decline and let the caller fall back to the correctly-rounded - // std::from_chars / std::strtod path. - static_cast(first); - static_cast(last); - static_cast(out); - return false; -#else - static const std::array powers_of_ten = + static constexpr int mantissa_bits() noexcept { - { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 - } - }; + return 23; + } + static constexpr int sign_bit() noexcept + { + return 31; + } + static constexpr int minimum_exponent() noexcept + { + return -127; + } + static constexpr int infinite_power() noexcept + { + return 0xFF; + } + // w * 10^q with w < 2^64 is below half the smallest subnormal number for + // q < smallest_power_of_ten() and at least infinity for q > largest_power_of_ten() + static constexpr int smallest_power_of_ten() noexcept + { + return -64; + } + static constexpr int largest_power_of_ten() noexcept + { + return 38; + } + // w * 10^q can only be exactly between two numbers for q in this range + static constexpr int min_exponent_round_to_even() noexcept + { + return -17; + } + static constexpr int max_exponent_round_to_even() noexcept + { + return 10; + } + // Clinger's fast path: w and 10^|q| are exact + static constexpr int max_exponent_fast_path() noexcept + { + return 10; + } + static constexpr std::uint64_t max_mantissa_fast_path() noexcept + { + return std::uint64_t{2} << 23u; + } + // a midpoint between two numbers has at most this many significant digits + static constexpr std::int64_t max_digits() noexcept + { + return 114; + } +}; - const char* p = first; - bool negative = false; - if (p != last && (*p == '-' || *p == '+')) - { - negative = (*p == '-'); - ++p; - } - - std::uint64_t significand = 0; - int num_digits = 0; - int fractional_digits = 0; - bool seen_dot = false; - bool any_digit = false; - for (; p != last; ++p) - { - const char c = *p; - if (c >= '0' && c <= '9') - { - any_digit = true; - if (JSON_HEDLEY_UNLIKELY(num_digits >= 19)) - { - return false; // significand may not fit into uint64_t - } - significand = (significand * 10u) + static_cast(c - '0'); - ++num_digits; - fractional_digits += static_cast(seen_dot); - } - else if (c == '.') - { - if (JSON_HEDLEY_UNLIKELY(seen_dot)) - { - return false; - } - seen_dot = true; - } - else if (c == 'e' || c == 'E') - { - ++p; - break; - } - else - { - return false; - } - } - if (JSON_HEDLEY_UNLIKELY(!any_digit)) - { - return false; - } - - int exponent = 0; - if (p != last) // an exponent part remains - { - bool exp_negative = false; - if (p != last && (*p == '-' || *p == '+')) - { - exp_negative = (*p == '-'); - ++p; - } - bool any_exp_digit = false; - for (; p != last; ++p) - { - if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9')) - { - return false; - } - exponent = (exponent * 10) + (*p - '0'); - any_exp_digit = true; - if (JSON_HEDLEY_UNLIKELY(exponent > 9999)) - { - return false; - } - } - if (JSON_HEDLEY_UNLIKELY(!any_exp_digit)) - { - return false; - } - if (exp_negative) - { - exponent = -exponent; - } - } - - const int scale = exponent - fractional_digits; - if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast(1) << 53))) - { - return false; // significand not exactly representable as double - } - - auto result = static_cast(significand); - if (scale >= 0) - { - if (JSON_HEDLEY_UNLIKELY(scale > 22)) - { - return false; - } - result *= powers_of_ten[static_cast(scale)]; - } - else - { - if (JSON_HEDLEY_UNLIKELY(-scale > 22)) - { - return false; - } - result /= powers_of_ten[static_cast(-scale)]; - } - out = negative ? -result : result; - return true; -#endif -} - -/// fast float path is only exact for `double`; decline for float/long double -template -bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept +template<> +struct ieee_binary_format<53> // binary64 { - return false; -} + static constexpr int mantissa_bits() noexcept + { + return 52; + } + static constexpr int sign_bit() noexcept + { + return 63; + } + static constexpr int minimum_exponent() noexcept + { + return -1023; + } + static constexpr int infinite_power() noexcept + { + return 0x7FF; + } + static constexpr int smallest_power_of_ten() noexcept + { + return -342; + } + static constexpr int largest_power_of_ten() noexcept + { + return 308; + } + static constexpr int min_exponent_round_to_even() noexcept + { + return -4; + } + static constexpr int max_exponent_round_to_even() noexcept + { + return 23; + } + static constexpr int max_exponent_fast_path() noexcept + { + return 22; + } + static constexpr std::uint64_t max_mantissa_fast_path() noexcept + { + return std::uint64_t{2} << 52u; + } + static constexpr std::int64_t max_digits() noexcept + { + return 769; + } +}; /*! -@brief parse a float with std::from_chars (Eisel-Lemire) when available +@brief whether @a FloatType is IEEE-754 binary32 or binary64 -std::from_chars is locale-independent, correctly rounded, and - via the -Eisel-Lemire algorithm in modern standard libraries - much faster than strtod -over the whole value range (not just the Clinger subset). It is used only when -__cpp_lib_to_chars indicates full floating-point support and only when it -consumes the entire token ([first, last)). An under-/overflow (result_out_of_range) also declines, so -the caller's strtod fallback supplies the well-defined ±inf/0 result the parser -expects (side-stepping the P4168 divergence between implementations). - -@return true if the value was parsed exactly and fully; false to fall back +These formats (float, double, and long double where it is binary64, e.g. +with MSVC or on Apple arm64) are converted by parse_float_native(). The +predicate is the one the serializer uses to choose Grisu2. */ template -bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +struct has_native_float_format { - // JSON_HAS_CPP_17 must gate the use as well as the include above: - // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even - // in C++14 mode, where is not included. -#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) - const auto result = std::from_chars(first, last, out); - return result.ec == std::errc() && result.ptr == last; -#else - static_cast(first); - static_cast(last); - static_cast(out); - return false; -#endif -} + static constexpr bool value = + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 24 && std::numeric_limits::max_exponent == 128) || + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && std::numeric_limits::max_exponent == 1024); +}; -/// whether the eight bytes of @a v (see read_eight_bytes()) are ASCII digits -/// (after fast_float's is_made_of_eight_digits_fast) -inline bool is_eight_digits(std::uint64_t v) noexcept -{ - return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u; -} +/// the C++ type (float or double) that holds a binary32 or binary64 @a FloatType +template +using native_float_t = typename std::conditional::digits == 24, float, double>::type; /// the value of the eight ASCII digits in @a v (see read_eight_bytes()), three -/// multiplications instead of eight (after simdjson and fast_float) -inline std::uint32_t parse_eight_digits(std::uint64_t v) noexcept +/// multiplications instead of eight (after simdjson and fast_float); always +/// inlined, as GCC otherwise calls it in the number loops +JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noexcept { v = ((v & 0x0F0F0F0F0F0F0F0Fu) * 2561u) >> 8u; v = ((v & 0x00FF00FF00FF00FFu) * 6553601u) >> 16u; return static_cast(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u); } +/// whether [first, last) contains a digit other than '0' +inline bool has_nonzero_digit(const char* first, const char* last) noexcept +{ + return std::any_of(first, last, [](char c) noexcept + { + return c != '0'; + }); +} + +/// the value of the validated exponent digits [+-]?[0-9]+ in [first, last), +/// saturated far beyond every range +inline std::int64_t parse_float_exponent(const char* first, const char* last) noexcept +{ + const bool negative = *first == '-'; + first += (*first == '-' || *first == '+') ? 1 : 0; + constexpr std::int64_t saturation = 100000000000000000; // 10^17 + std::int64_t value = 0; + for (; first != last; ++first) + { + if (value < saturation) + { + value = (value * 10) + (*first - '0'); + } + } + return negative ? -value : value; +} + +/// a float token as w * 10^exponent, see parse_float_significand() +struct float_significand +{ + std::uint64_t w = 0; ///< the first (at most 19) significant digits + std::int64_t exponent = 0; ///< the decimal exponent of the last digit in w + bool negative = false; ///< whether the token starts with '-' + bool truncated = false; ///< whether nonzero digits follow the ones in w +}; + /*! -@brief the double nearest to w * 10^q (Eisel-Lemire) +@brief split a validated number token into sign, significand, and exponent + +The lexer has validated the token against the JSON grammar and knows where its +parts are, so this needs no character classification: the integer part ends at +@a decimal_point_position (or @a mantissa_end), the fraction at @a mantissa_end, +and an exponent follows. At most 19 significant digits are kept; the value then +lies in [w, w + 1) * 10^exponent, and is exactly w * 10^exponent unless +truncated is set. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +*/ +inline float_significand parse_float_significand(const char* first, const char* last, + std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +{ + float_significand s; + const char* p = first; + s.negative = *p == '-'; + p += s.negative ? 1 : 0; + const bool has_dot = decimal_point_position != std::string::npos; + const char* const mantissa_last = first + mantissa_end; + const char* const integer_last = has_dot ? first + decimal_point_position : mantissa_last; + + std::uint64_t w = 0; + int remaining = 19; // digits that still fit into w + if (*p != '0') // the integer part is "0" or [1-9][0-9]* + { + while (remaining >= 8 && integer_last - p >= 8) + { + w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p)); + p += 8; + remaining -= 8; + } + for (; remaining > 0 && p != integer_last; ++p, --remaining) + { + w = (w * 10u) + static_cast(*p - '0'); + } + s.exponent = integer_last - p; + s.truncated = has_nonzero_digit(p, integer_last); + } + + if (has_dot) + { + p = integer_last + 1; + if (w == 0) + { + // zeros after the decimal point of "0." are not significant + const char* const zeros = p; + while (p != mantissa_last && *p == '0') + { + ++p; + } + s.exponent -= p - zeros; + } + const char* const digits = p; + while (remaining >= 8 && mantissa_last - p >= 8) + { + w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p)); + p += 8; + remaining -= 8; + } + for (; remaining > 0 && p != mantissa_last; ++p, --remaining) + { + w = (w * 10u) + static_cast(*p - '0'); + } + s.exponent -= p - digits; + s.truncated = s.truncated || has_nonzero_digit(p, mantissa_last); + } + + if (mantissa_last != last) + { + s.exponent += parse_float_exponent(mantissa_last + 1, last); + } + s.w = w; + return s; +} + +/*! +@brief the bits of the float nearest to w * 10^q (Eisel-Lemire) The algorithm of Daniel Lemire, "Number Parsing at a Gigabyte per Second" (Software: Practice and Experience, 2021), after fast_float's compute_float (used under the MIT license). With a 128-bit approximation of 5^q, the product -is always sufficient to round correctly for w with at most 19 digits (Noble -Mushtak and Daniel Lemire, "Fast number parsing without fallback", Software: -Practice and Experience, 2023). Only integer arithmetic is used, so the result -does not depend on the floating-point environment. +is always sufficient to round correctly for w < 2^64 (Noble Mushtak and Daniel +Lemire, "Fast number parsing without fallback", Software: Practice and +Experience, 2023). Only integer arithmetic is used, so the result does not +depend on the floating-point environment. +It is always inlined, like decimal_to_float(), so that hot loops of callers +keep the whole conversion inline. + +@tparam Format ieee_binary_format<24> (binary32) or ieee_binary_format<53> (binary64) @param[in] q decimal exponent -@param[in] w significand, w != 0 +@param[in] w significand @return the IEEE-754 bits of the positive result (0 for underflow, infinity for overflow) */ -inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept +template +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept { - constexpr int mantissa_bits = 52; - constexpr std::uint64_t infinity = std::uint64_t{0x7FF} << mantissa_bits; - if (q < pow5_128_smallest_power) + constexpr int mantissa_bits = Format::mantissa_bits(); + constexpr std::uint64_t infinity = static_cast(Format::infinite_power()) << mantissa_bits; + if (w == 0 || q < Format::smallest_power_of_ten()) { return 0; } - if (q > pow5_128_largest_power) + if (q > Format::largest_power_of_ten()) { return infinity; } @@ -365,8 +445,8 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept const auto upperbit = static_cast(product.high >> 63u); const int shift = upperbit + 64 - mantissa_bits - 3; std::uint64_t mantissa = product.high >> static_cast(shift); - // floor(log2(10^q)) + 63 + 1023, with log2(10) ~ 217706 / 2^16 - std::int64_t power2 = (((152170 + 65536) * q) >> 16) + 63 + upperbit - lz + 1023; + // floor(log2(10^q)) + 63 + bias, with log2(10) ~ 217706 / 2^16 + std::int64_t power2 = (((152170 + 65536) * q) >> 16) + 63 + upperbit - lz - Format::minimum_exponent(); if (power2 <= 0) // subnormal { @@ -375,17 +455,18 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept return 0; } mantissa >>= static_cast(-power2 + 1); + // no tie is possible here: that needs a small |q| mantissa += (mantissa & 1u); mantissa >>= 1u; // rounding up may produce the smallest normal number power2 = (mantissa < (std::uint64_t{1} << mantissa_bits)) ? 0 : 1; - return mantissa | (static_cast(power2) << mantissa_bits); + return (mantissa & ((std::uint64_t{1} << mantissa_bits) - 1)) | (static_cast(power2) << mantissa_bits); } - // a value exactly between two doubles rounds to even; this can only + // a value exactly between two floats rounds to even; this can only // happen for small |q|, where 5^q is exact - if (product.low <= 1 && q >= -4 && q <= 23 && (mantissa & 3u) == 1 - && (mantissa << static_cast(shift)) == product.high) + if (product.low <= 1 && q >= Format::min_exponent_round_to_even() && q <= Format::max_exponent_round_to_even() + && (mantissa & 3u) == 1 && (mantissa << static_cast(shift)) == product.high) { mantissa &= ~std::uint64_t{1}; } @@ -397,198 +478,420 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept ++power2; } mantissa &= ~(std::uint64_t{1} << mantissa_bits); - if (power2 >= 0x7FF) + if (power2 >= Format::infinite_power()) { return infinity; } return mantissa | (static_cast(power2) << mantissa_bits); } -/*! -@brief parse a validated float token with the Eisel-Lemire algorithm - -The significand is accumulated eight digits at a time where possible. A token -with more than 19 significant digits is truncated to w; the value then lies -in [w, w + 1) * 10^q, and it is only returned if both ends round to the same -double, which covers all but a few such tokens. - -@param[in] first pointer to the first character of the token -@param[in] last pointer past the last character -@param[out] out the correctly rounded value on success (±infinity if it - overflows, like strtod) -@return true on success; false if strtod must decide -*/ -inline bool parse_float_eisel_lemire(const char* first, const char* last, double& out) noexcept +/// an unsigned integer of up to 4096 bits for digit_comparison() (32-bit limbs, +/// so only 32x32->64-bit multiplications are needed) +class float_bigint { - const char* p = first; - const bool negative = (p != last && *p == '-'); - if (negative) + public: + explicit float_bigint(std::uint64_t value) noexcept { - ++p; + for (; value != 0; value >>= 32u) + { + limbs[count++] = static_cast(value); + } } - std::uint64_t w = 0; - unsigned int digits = 0; // significant digits in w - std::int64_t exponent = 0; - bool truncated = false; - bool in_fraction = false; - for (;;) + /// *this = *this * factor + summand + void multiply_add(std::uint32_t factor, std::uint32_t summand) noexcept { - // eight digits at a time, as long as they fit into w - while (w != 0 && digits <= 19u - 8u && last - p >= 8) + std::uint64_t carry = summand; + for (std::size_t i = 0; i < count; ++i) { - const std::uint64_t v = read_eight_bytes(p); - if (!is_eight_digits(v)) - { - break; - } - w = (w * 100000000u) + parse_eight_digits(v); - digits += 8u; - exponent -= in_fraction ? 8 : 0; - p += 8; + const std::uint64_t product = (static_cast(limbs[i]) * factor) + carry; + limbs[i] = static_cast(product); + carry = product >> 32u; } - if (p == last) + if (carry != 0) { - break; + JSON_ASSERT(count < limbs.size()); + limbs[count++] = static_cast(carry); } - const char c = *p; - if (c >= '0' && c <= '9') + } + + /// *this = *this * 5^n + void multiply_power_of_five(std::int64_t n) noexcept + { + static const std::array powers = { - if (w == 0 && c == '0') - { - // leading zeros are not significant, but scale a fraction - exponent -= in_fraction ? 1 : 0; - } - else if (digits < 19u) - { - w = (w * 10u) + static_cast(c - '0'); - ++digits; - exponent -= in_fraction ? 1 : 0; - } - else - { - // dropped: the value lies between w and w + 1 (in units of - // the last kept digit) unless all dropped digits are zero - truncated = truncated || c != '0'; - exponent += in_fraction ? 0 : 1; - } - ++p; + {1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u} + }; + for (; n >= 13; n -= 13) + { + multiply_add(powers[13], 0); } - else if (c == '.') + multiply_add(powers[static_cast(n)], 0); + } + + /// *this = *this * 2^n + void shift_left(std::int64_t n) noexcept + { + if (count == 0) { - in_fraction = true; - ++p; + return; + } + const auto limb_shift = static_cast(n / 32); + const auto bit_shift = static_cast(n % 32); + JSON_ASSERT(count + limb_shift + 1 <= limbs.size()); + if (bit_shift != 0) + { + std::uint32_t carry = 0; + for (std::size_t i = 0; i < count; ++i) + { + const std::uint32_t limb = limbs[i]; + limbs[i] = (limb << bit_shift) | carry; + carry = limb >> (32u - bit_shift); + } + if (carry != 0) + { + limbs[count++] = carry; + } + } + if (limb_shift != 0) + { + for (std::size_t i = count; i-- > 0;) + { + limbs[i + limb_shift] = limbs[i]; + } + for (std::size_t i = 0; i < limb_shift; ++i) + { + limbs[i] = 0; + } + count += limb_shift; + } + } + + /// -1, 0, or 1 if *this is less than, equal to, or greater than @a other + int compare(const float_bigint& other) const noexcept + { + if (count != other.count) + { + return count < other.count ? -1 : 1; + } + for (std::size_t i = count; i-- > 0;) + { + if (limbs[i] != other.limbs[i]) + { + return limbs[i] < other.limbs[i] ? -1 : 1; + } + } + return 0; + } + + private: + std::array limbs{{}}; + std::size_t count = 0; +}; + +/*! +@brief round a token exactly when eisel_lemire() cannot decide (slow path) + +The value v of the token lies strictly between two adjacent floats, whose +lower one has the bits @a lower, and the result depends on whether v is below, +at, or above the midpoint m between them. Both are compared exactly as big +integers: v = D * 10^s with the significant digits D (at most +Format::max_digits() of them, more than any midpoint has; further nonzero +digits only put v above m) and m = (2 * mantissa + 1) * 2^(e - 1). This is the +digit comparison of fast_float (Daniel Lemire and contributors, used under the +MIT license), simplified by starting from the two candidates. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] lower the bits of the float below v +@return the bits of the correctly rounded result +*/ +template +std::uint64_t digit_comparison(const char* first, const char* last, std::uint64_t lower) noexcept +{ + const char* p = first + ((*first == '-') ? 1 : 0); + + // D, in chunks of up to 9 digits, and s + float_bigint digits(0); + std::int64_t count = 0; + std::int64_t point = 0; // the value is 0.D... * 10^point + bool truncated = false; + std::uint32_t chunk = 0; + int chunk_digits = 0; + static const std::array powers_of_ten = {{1u, 10u, 100u, 1000u, 10000u, 100000u, 1000000u, 10000000u, 100000000u, 1000000000u}}; + const auto append = [&](char c) noexcept + { + if (count < Format::max_digits()) + { + chunk = (chunk * 10u) + static_cast(c - '0'); + ++count; + if (++chunk_digits == 9) + { + digits.multiply_add(powers_of_ten[9], chunk); + chunk = 0; + chunk_digits = 0; + } } else { - break; // 'e' or 'E' + truncated = truncated || c != '0'; + } + }; + bool significant = false; + for (; p != last && *p >= '0' && *p <= '9'; ++p) + { + significant = significant || *p != '0'; + if (significant) + { + append(*p); + ++point; } } - - if (p != last) + if (p != last && *p == '.') { - ++p; // 'e' or 'E' - bool exp_negative = false; - if (p != last && (*p == '-' || *p == '+')) + for (++p; p != last && *p >= '0' && *p <= '9'; ++p) { - exp_negative = (*p == '-'); - ++p; - } - std::int64_t exp_value = 0; - for (; p != last; ++p) - { - // saturate: any exponent beyond this under- or overflows anyway - if (exp_value < 100000) + significant = significant || *p != '0'; + if (significant) { - exp_value = (exp_value * 10) + (*p - '0'); + append(*p); + } + else + { + --point; } } - exponent += exp_negative ? -exp_value : exp_value; } - - std::uint64_t bits = 0; - if (w != 0) + if (chunk_digits != 0) { - bits = eisel_lemire(exponent, w); - if (truncated && (w + 1 == 0 || eisel_lemire(exponent, w + 1) != bits)) - { - return false; - } + digits.multiply_add(powers_of_ten[static_cast(chunk_digits)], chunk); } - bits |= negative ? (std::uint64_t{1} << 63u) : 0u; - static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); - std::memcpy(&out, &bits, sizeof(out)); - return true; + if (p != last) + { + point += parse_float_exponent(p + 1, last); + } + const std::int64_t s = point - count; // v = D * 10^s + + // the midpoint above the lower candidate + constexpr int mantissa_bits = Format::mantissa_bits(); + const std::uint64_t exponent_field = lower >> mantissa_bits; + std::uint64_t mantissa = lower & ((std::uint64_t{1} << mantissa_bits) - 1); + std::int64_t e = 1 + Format::minimum_exponent() - mantissa_bits; // of the smallest subnormal number + if (exponent_field != 0) + { + mantissa |= std::uint64_t{1} << mantissa_bits; + e += static_cast(exponent_field) - 1; + } + float_bigint midpoint((2 * mantissa) + 1); + const std::int64_t midpoint_exponent = e - 1; // m = midpoint * 2^midpoint_exponent + + // compare D * 5^s * 2^s with midpoint * 2^midpoint_exponent + if (s >= 0) + { + digits.multiply_power_of_five(s); + } + else + { + midpoint.multiply_power_of_five(-s); + } + const std::int64_t shift = s - midpoint_exponent; + if (shift >= 0) + { + digits.shift_left(shift); + } + else + { + midpoint.shift_left(-shift); + } + const int order = digits.compare(midpoint); + const bool round_up = order > 0 || (order == 0 && (truncated || (mantissa & 1u) != 0)); + return lower + (round_up ? 1u : 0u); } -/// Eisel-Lemire is only implemented for `double` -template -bool parse_float_eisel_lemire(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept +/// the double with the IEEE-754 bits @a bits +inline void float_from_bits(std::uint64_t bits, double& value) noexcept { - return false; + static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); + std::memcpy(&value, &bits, sizeof(value)); +} + +/// the float with the IEEE-754 bits @a bits (the lower 32) +inline void float_from_bits(std::uint64_t bits, float& value) noexcept +{ + static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits"); + const auto bits32 = static_cast(bits); + std::memcpy(&value, &bits32, sizeof(bits32)); +} + +/// the powers of ten that are exact in binary64 (up to 10^22) +inline double exact_power_of_ten(std::int64_t n, double /*tag*/) noexcept +{ + static const std::array powers = + { + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 + } + }; + return powers[static_cast(n)]; +} + +/// the powers of ten that are exact in binary32 (up to 10^10) +inline float exact_power_of_ten(std::int64_t n, float /*tag*/) noexcept +{ + static const std::array powers = + { + {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f, 1e7f, 1e8f, 1e9f, 1e10f} + }; + return powers[static_cast(n)]; } /*! -@brief check whether Clinger's fast path can still succeed for a float token +@brief the binary32/binary64 value of a significand that was not truncated -parse_float_fast() needs a significand below 2^53. A mantissa with 17 or -more significant digits is at least 10^16 and therefore always exceeds it, -so calling the fast path would walk the token one extra time only to -decline before strtod has to run anyway. +The result is (-1)^negative * w * 10^exponent, correctly rounded (ties to +even): with Clinger's fast path where w and 10^|exponent| are exact, so that a +single floating-point operation rounds (only where intermediate results are not +kept in extended precision, see FLT_EVAL_METHOD), and with eisel_lemire() +otherwise. A value too large for the type becomes ±infinity, a value too small +±0. -Significant digits are the mantissa's digits from the first nonzero one on; -the sign, the decimal point, leading zeros, and the exponent do not count. -The answer is derived from indices - the digits are not scanned again - so -this stays off the hot path of the number scanners. +This is the core of the conversion that other parsers of JSON text share: they +can split a token themselves and still get the lexer's result. It is always +inlined, so that their hot loops keep the whole conversion inline. -@param[in] token the validated number token ('.' as decimal point) -@param[in] decimal_point_position index of the '.' in @a token, or - std::string::npos if there is none -@param[in] mantissa_end offset just past the last mantissa byte -@return false if parse_float_fast() is guaranteed to decline +@param[in] s the significand, with s.truncated == false */ -inline bool mantissa_fits_clinger(const char* token, std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +template +JSON_HEDLEY_ALWAYS_INLINE FloatType decimal_to_float(const float_significand& s) noexcept { - // 10^16 already exceeds 2^53, so 17 digits can never fit - constexpr std::size_t limit = 17; + using result_type = native_float_t; + using format = ieee_binary_format::digits>; + JSON_ASSERT(!s.truncated); - const std::size_t neg = (token[0] == '-') ? 1u : 0u; - const std::size_t has_dot = (decimal_point_position != std::string::npos) ? 1u : 0u; - // the JSON grammar restricts the integer part to "0" or [1-9][0-9]*, so - // a leading zero can only be a lone "0", which is not significant - const std::size_t lead_zero = (token[neg] == '0') ? 1u : 0u; - JSON_ASSERT(mantissa_end >= neg + has_dot + lead_zero); - std::size_t digits = mantissa_end - neg - has_dot - lead_zero; - - if (JSON_HEDLEY_LIKELY(digits < limit)) +#if !defined(FLT_EVAL_METHOD) || FLT_EVAL_METHOD == 0 + if (s.exponent >= -format::max_exponent_fast_path() && s.exponent <= format::max_exponent_fast_path() + && s.w <= format::max_mantissa_fast_path()) { - return true; - } - - // Only a number below 1 can carry further insignificant zeros, and only - // while the count stays at the limit does removing them change the - // answer - so this loop is skipped for all but a few tokens. The - // fraction is located through decimal_point_position rather than by - // searching '.'. - if (lead_zero != 0) - { - JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit - for (std::size_t i = decimal_point_position + 1; - digits >= limit && i < mantissa_end && token[i] == '0'; ++i) + auto value = static_cast(s.w); + if (s.exponent < 0) { - --digits; + value /= exact_power_of_ten(-s.exponent, result_type{}); } + else + { + value *= exact_power_of_ten(s.exponent, result_type{}); + } + const FloatType result = s.negative ? -value : value; + return result; + } +#endif + + result_type value{}; + float_from_bits(eisel_lemire(s.exponent, s.w) | (s.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + const FloatType result = value; + return result; +} + +/*! +@brief convert a validated number token to the nearest binary32/binary64 value + +The conversion is correctly rounded (ties to even) and independent of the +locale and of the C and C++ libraries: +1. parse_float_significand() splits the token into w * 10^q. +2. If no digits were dropped, decimal_to_float() rounds w * 10^q (Clinger's + fast path or Eisel-Lemire). +3. Otherwise, the value lies in [w, w + 1) * 10^q: if eisel_lemire() rounds + both ends to the same value, so does the token (in all but rare cases). +4. Otherwise, digit_comparison() compares the token exactly with the midpoint + between the two candidates. +A value too large for the type becomes ±infinity (the parser reports +out_of_range.406), a value too small ±0. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +*/ +template +FloatType parse_float_native(const char* first, const char* last, + std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +{ + using result_type = native_float_t; + using format = ieee_binary_format::digits>; + static_assert(std::numeric_limits::digits == std::numeric_limits::digits, "unexpected float format"); + + const float_significand s = parse_float_significand(first, last, decimal_point_position, mantissa_end); + if (JSON_HEDLEY_LIKELY(!s.truncated)) + { + return decimal_to_float(s); } - return digits < limit; + std::uint64_t bits = eisel_lemire(s.exponent, s.w); + if (JSON_HEDLEY_UNLIKELY(bits != eisel_lemire(s.exponent, s.w + 1))) + { + bits = digit_comparison(first, last, bits); + } + result_type value{}; + float_from_bits(bits | (s.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + const FloatType result = value; + return result; +} + +/*! +@brief parse a float with std::from_chars when available + +Only used for the formats parse_float_native() does not convert (long double +formats other than binary64). std::from_chars is locale-independent and +correctly rounded. It is used only when __cpp_lib_to_chars indicates full +floating-point support and only when it consumes the entire token ([first, +last)). An under-/overflow (result_out_of_range) also declines, so the +caller's strtold fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ + // JSON_HAS_CPP_17 must gate the use as well as the include above: + // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even + // in C++14 mode, where is not included. +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + +/// binary32 and binary64: the library's own conversion, which always succeeds +template +bool convert_float_fast(const char* first, const char* last, std::size_t decimal_point_position, + std::size_t mantissa_end, FloatType& value, std::true_type /*native*/) noexcept +{ + value = parse_float_native(first, last, decimal_point_position, mantissa_end); + return true; +} + +/// other formats (long double on x87, binary128, double-double): std::from_chars, if available +template +bool convert_float_fast(const char* first, const char* last, std::size_t /*decimal_point_position*/, + std::size_t /*mantissa_end*/, FloatType& value, std::false_type /*native*/) noexcept +{ + return parse_float_from_chars(first, last, value); } /*! @brief convert a validated float token without the C library, if possible -Tries std::from_chars (when available), Clinger's exact fast path (double -only, skipped when it cannot succeed), and the Eisel-Lemire algorithm (double -only). +float, double, and long double where it is binary64 are always converted, by +parse_float_native(). Other long double formats are converted with +std::from_chars where the standard library supports it. @param[in] first pointer to the first character of the token @param[in] last pointer past the last character @@ -604,19 +907,8 @@ template bool convert_float_fast(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end, FloatType& value) noexcept { - if (parse_float_from_chars(first, last, value)) - { - return true; - } - // Skipping a fast path that cannot succeed is lossless and saves a full - // extra pass over the token's bytes, which otherwise shows up on - // high-precision inputs such as canada.json - if (mantissa_fits_clinger(first, decimal_point_position, mantissa_end) - && parse_float_fast(first, last, value)) - { - return true; - } - return parse_float_eisel_lemire(first, last, value); + return convert_float_fast(first, last, decimal_point_position, mantissa_end, value, + std::integral_constant::value> {}); } /// std::strtof, std::strtod, or std::strtold, chosen by the type of @a f @@ -640,36 +932,45 @@ inline void strtof_by_type(long double& f, const char* str, char** endptr) noexc f = std::strtold(str, endptr); } -/// return the decimal point of the current locale -inline char get_decimal_point() noexcept +/// return the decimal point of the current locale (it may be longer than one byte) +inline std::string get_decimal_point() { const auto* loc = localeconv(); JSON_ASSERT(loc != nullptr); - return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + return (loc->decimal_point == nullptr || *loc->decimal_point == '\0') ? "." : loc->decimal_point; } /*! @brief convert a validated float token with strtof/strtod/strtold +Only used for what convert_float_fast() does not convert: long double formats +other than binary64 where std::from_chars is unavailable or reports an under- +or overflow, and floating-point types that are not IEEE-754 (see +has_native_float_format). + These functions expect the decimal point of the *current* locale, so it is looked up right before the conversion instead of once when the lexer is constructed: a locale change in between (by a parser callback, a SAX -handler, or another thread) must not truncate the value (#5198). The -token has been validated before, so if the conversion stops early and the -decimal point changed in the meantime, the locale changed between the +handler, or another thread) must not truncate the value (#5198). A +single-byte decimal point is substituted in place and restored afterwards, +because the token is also handed to the SAX interface. A longer one (e.g., +the two-byte U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) is put into a copy of the +token instead (#5660). + +The token has been validated before, so if the conversion stops early and +the decimal point changed in the meantime, the locale changed between the lookup and the call, and the conversion is repeated with the new decimal -point. If the decimal point did not change, a retry cannot succeed: the -locale's decimal point is not a single character (e.g., the two-byte -U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) and cannot be substituted in place. -The value strtod parsed up to that point is kept, as before this change. +point. If it did not change, the value strtod parsed up to that point is +kept. Note that changing the locale in another thread *while* strtod runs is undefined behavior of the C library, which this function cannot prevent. -@param[in,out] token the token with '.' as decimal point; its - decimal point is replaced during the - conversion and restored afterwards - (data() must be NUL-terminated) +@param[in,out] token the token with '.' as decimal point; a + single-byte decimal point is put in + place during the conversion and + restored afterwards (data() must be + NUL-terminated) @param[in] decimal_point_position index of the '.' in @a token, or std::string::npos if there is none @param[out] value the converted value @@ -678,38 +979,76 @@ template void convert_float_locale_aware(StringType& token, std::size_t decimal_point_position, FloatType& value) { const bool has_dot = decimal_point_position != std::string::npos; - char decimal_point = get_decimal_point(); + std::string decimal_point = get_decimal_point(); for (;;) { - const bool substitute = has_dot && decimal_point != '.'; - if (substitute) - { - token[decimal_point_position] = static_cast(decimal_point); - } - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - strtof_by_type(value, token.data(), &endptr); - - if (substitute) + bool complete = false; + if (!has_dot || decimal_point.size() == 1) { - // the caller hands the token on (e.g. to the SAX interface) with '.' - token[decimal_point_position] = '.'; + const bool substitute = has_dot && decimal_point[0] != '.'; + if (substitute) + { + token[decimal_point_position] = static_cast(decimal_point[0]); + } + strtof_by_type(value, token.data(), &endptr); + if (substitute) + { + // the caller hands the token on (e.g. to the SAX interface) with '.' + token[decimal_point_position] = '.'; + } + complete = endptr == token.data() + token.size(); + } + else + { + std::string buffer(token.data(), token.size()); + buffer.replace(decimal_point_position, 1, decimal_point); + strtof_by_type(value, buffer.data(), &endptr); + complete = endptr == buffer.data() + buffer.size(); } - if (JSON_HEDLEY_LIKELY(endptr == token.data() + token.size())) + if (JSON_HEDLEY_LIKELY(complete)) { return; } // retry only if the locale changed; otherwise, this would loop forever - const char current_decimal_point = get_decimal_point(); + std::string current_decimal_point = get_decimal_point(); if (current_decimal_point == decimal_point) { return; } - decimal_point = current_decimal_point; + decimal_point = std::move(current_decimal_point); } } +/*! +@brief convert a validated float token like the lexer does + +For parsers of JSON text other than the lexer, which converts its own token +buffer in place. float, double, and long double where it is binary64 are +converted without allocation and independent of the locale; only other long +double formats that std::from_chars does not support need a copy of the token +for convert_float_locale_aware(). + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +@return the value, ±infinity if it overflows +*/ +template +FloatType convert_float(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end) +{ + FloatType value{}; + if (!convert_float_fast(first, last, decimal_point_position, mantissa_end, value)) + { + std::string token(first, last); + convert_float_locale_aware(token, decimal_point_position, value); + } + return value; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 58e06a499..9fda2442f 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -451,7 +451,7 @@ class parser } case token_type::end_of_input: { - if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) + if (JSON_HEDLEY_UNLIKELY(m_lexer.get_chars_read_total() == 1)) { // there is nothing to recover static_cast(report_error(sax, parse_error::create(exception_id::syntax_error, m_lexer.get_position(), @@ -1216,8 +1216,10 @@ class parser { // MSVC 2015 reports C4100 (unreferenced parameter) when SAX::parse_error is static static_cast(sax); - return sax.parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(exception_id::syntax_error, m_lexer.get_position(), message, nullptr)); + // computing the position is O(input read so far) for seekable input + const position_t position = m_lexer.get_position(); + return sax.parse_error(position, m_lexer.get_token_string(), + parse_error::create(exception_id::syntax_error, position, message, nullptr)); } std::string exception_message(const token_type expected, const std::string& context) diff --git a/include/nlohmann/detail/input/pow5_table.hpp b/include/nlohmann/detail/input/pow5_table.hpp index 496fdb2f7..c7170e903 100644 --- a/include/nlohmann/detail/input/pow5_table.hpp +++ b/include/nlohmann/detail/input/pow5_table.hpp @@ -22,6 +22,10 @@ namespace detail constexpr std::int64_t pow5_128_smallest_power = -342; constexpr std::int64_t pow5_128_largest_power = 308; +// every entry of pow5_128() holds two 64-bit halves of 5^q, one per covered power of 5 +static_assert((pow5_128_largest_power - pow5_128_smallest_power + 1) * 2 == 1302, + "pow5_128_smallest_power/pow5_128_largest_power must match the size of the pow5_128() table"); + /*! @brief 128-bit approximations of 5^q for q in [-342, 308] diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index 8a6f03122..fd2289b47 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -8,10 +8,12 @@ #pragma once +#include // array #include // size_t -#include // uint64_t +#include // uint64_t, uint8_t #include // memcpy +#include #include // Optional SIMD backend for bulk UTF-8 validation. This is an opt-in external @@ -69,18 +71,12 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t word = 0; - std::memcpy(&word, data + i, sizeof(word)); - if (swar_string_special(word) != 0) + const std::uint64_t special = swar_string_special(read_eight_bytes(data + i)); + if (special != 0) { - // a special byte is in this word; locate it (endian-agnostic) - for (std::size_t j = 0; j < 8; ++j) - { - if (is_string_special(data[i + j])) - { - return i + j; - } - } + // the lowest flagged byte is the first special one: the borrows of + // the subtractions can only flag bytes above a true hit + return i + (static_cast(count_trailing_zeros(special)) / 8); } } for (; i < n; ++i) @@ -114,8 +110,7 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) @@ -126,7 +121,9 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ | (v & high); // >= 0x80 if (stop != 0) { - break; + // the lowest flagged byte is the first one to stop at (see + // find_string_special()) + return i + (static_cast(count_trailing_zeros(stop)) / 8); } } for (; i < n; ++i) @@ -253,12 +250,18 @@ inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t { break; // end of buffer, or a quote/escape/control byte } - const std::size_t seq = validate_one_utf8(data + pos, n - pos); - if (seq == 0) + // a run of multi-byte sequences (e.g. CJK text) is validated sequence + // by sequence without searching for the next special byte in between + do { - break; // ill-formed or truncated: let the byte path diagnose it + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + return pos; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; } - pos += seq; + while (pos < n && data[pos] >= 0x80u); } return pos; } @@ -273,8 +276,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; const std::uint64_t hit = ((q - ones) & ~q & high) @@ -282,14 +284,8 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t | ((v - 0x2020202020202020ull) & ~v & high); if (hit != 0) { - for (std::size_t j = 0; j < 8; ++j) - { - const unsigned char c = data[i + j]; - if (c == '\"' || c == '\\' || c < 0x20u) - { - return i + j; - } - } + // the lowest flagged byte is the first delimiter (see find_string_special()) + return i + (static_cast(count_trailing_zeros(hit)) / 8); } } for (; i < n; ++i) @@ -320,5 +316,50 @@ inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noe return scalar_string_bulk_run(data, n); } +// Decode the 4 hex digits at [data, data+4) - the digits following a `\u` +// escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte +// (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a +// hex digit ('0'..'9', 'A'..'F', 'a'..'f'). The caller must already have +// checked that 4 bytes are available; used by lexer::get_codepoint()'s +// contiguous fast path. On -1 it falls back to the byte-at-a-time loop, which +// stops at the first invalid digit, so the reported error and position are +// unaffected by this fast path. +inline int hex_codepoint(const unsigned char* data) noexcept +{ + static const std::array hex_digit_table = // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + { + { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 00..0F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 10..1F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 20..2F + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 30..3F ('0'..'9') + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 40..4F ('A'..'F') + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 50..5F + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 60..6F ('a'..'f') + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 70..7F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 80..8F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 90..9F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // A0..AF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // B0..BF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // C0..CF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // D0..DF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // E0..EF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // F0..FF + } + }; + + const std::uint8_t d0 = hex_digit_table[data[0]]; + const std::uint8_t d1 = hex_digit_table[data[1]]; + const std::uint8_t d2 = hex_digit_table[data[2]]; + const std::uint8_t d3 = hex_digit_table[data[3]]; + // every valid digit is <= 0xF; the combined OR only exceeds it if at + // least one of the four bytes was not a hex digit (looked up as 0xFF) + if ((d0 | d1 | d2 | d3) > 0x0F) + { + return -1; + } + return (d0 << 12) | (d1 << 8) | (d2 << 4) | d3; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/iterators/iter_impl.hpp b/include/nlohmann/detail/iterators/iter_impl.hpp index 52b459ad2..b7c1df24a 100644 --- a/include/nlohmann/detail/iterators/iter_impl.hpp +++ b/include/nlohmann/detail/iterators/iter_impl.hpp @@ -280,7 +280,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci } /// @throw invalid_iterator.214 always - JSON_HEDLEY_NO_RETURN void throw_cannot_get_value() const + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_cannot_get_value() const { JSON_THROW(invalid_iterator::create(exception_id::iterator_value_unavailable, "cannot get value", m_object)); } @@ -294,40 +294,25 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); - return m_it.object_iterator->second; - } - - case value_t::array: - { - JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); - return *m_it.array_iterator; - } - - case value_t::null: - throw_cannot_get_value(); - - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) - { - return *m_object; - } - - throw_cannot_get_value(); - } + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); + return *m_it.array_iterator; } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); + return m_it.object_iterator->second; + } + + // neither array nor object: a primitive value (null has no value to point to) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type != value_t::null && m_it.primitive_iterator.is_begin())) + { + return *m_object; + } + + throw_cannot_get_value(); } /*! @@ -338,38 +323,25 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); - return &(m_it.object_iterator->second); - } - - case value_t::array: - { - JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); - return &*m_it.array_iterator; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) - { - return m_object; - } - - throw_cannot_get_value(); - } + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); + return &*m_it.array_iterator; } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); + return &(m_it.object_iterator->second); + } + + // neither array nor object: a primitive value + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return m_object; + } + + throw_cannot_get_value(); } /*! @@ -391,35 +363,20 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - std::advance(m_it.object_iterator, 1); - break; - } - - case value_t::array: - { - std::advance(m_it.array_iterator, 1); - break; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - ++m_it.primitive_iterator; - break; - } + ++m_it.array_iterator; + return *this; } + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + std::advance(m_it.object_iterator, 1); + return *this; + } + + // neither array nor object: a primitive value + ++m_it.primitive_iterator; return *this; } @@ -442,35 +399,20 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - std::advance(m_it.object_iterator, -1); - break; - } - - case value_t::array: - { - std::advance(m_it.array_iterator, -1); - break; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - --m_it.primitive_iterator; - break; - } + --m_it.array_iterator; + return *this; } + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + std::advance(m_it.object_iterator, -1); + return *this; + } + + // neither array nor object: a primitive value + --m_it.primitive_iterator; return *this; } @@ -484,7 +426,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci // if objects are not the same, the comparison is undefined if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) { - JSON_THROW(invalid_iterator::create(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object)); + throw_literal(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object); } // value-initialized forward iterators can be compared, and must compare equal to other value-initialized iterators of the same type #4493 @@ -493,25 +435,18 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci return true; } - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - return (m_it.object_iterator == other.m_it.object_iterator); - - case value_t::array: - return (m_it.array_iterator == other.m_it.array_iterator); - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - return (m_it.primitive_iterator == other.m_it.primitive_iterator); + return (m_it.array_iterator == other.m_it.array_iterator); } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + // neither array nor object: a primitive value + return (m_it.primitive_iterator == other.m_it.primitive_iterator); } /*! @@ -533,7 +468,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci // if objects are not the same, the comparison is undefined if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) { - JSON_THROW(invalid_iterator::create(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object)); + throw_literal(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object); } // value-initialized forward iterators can be compared, and must compare equal to other value-initialized iterators of the same type #4493 @@ -546,7 +481,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_order_on_object, "cannot compare order of object iterators", m_object)); + throw_literal(exception_id::iterator_order_on_object, "cannot compare order of object iterators", m_object); case value_t::array: return (m_it.array_iterator < other.m_it.array_iterator); @@ -602,7 +537,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object)); + throw_literal(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object); case value_t::array: { @@ -681,7 +616,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object)); + throw_literal(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object); case value_t::array: return m_it.array_iterator - other.m_it.array_iterator; @@ -710,7 +645,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_subscript_on_object, "cannot use operator[] for object iterators", m_object)); + throw_literal(exception_id::iterator_subscript_on_object, "cannot use operator[] for object iterators", m_object); case value_t::array: return *std::next(m_it.array_iterator, n); @@ -750,7 +685,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci return m_it.object_iterator->first; } - JSON_THROW(invalid_iterator::create(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object)); + throw_literal(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object); } /*! diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 1b439f9bf..a752deb08 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -16,14 +16,13 @@ #include // ostream #endif // JSON_NO_IO #include // max -#include // map -#include // accumulate #include // string #include // move #include // vector #include #include +#include #include #include #include @@ -69,12 +68,21 @@ class json_pointer /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ string_t to_string() const { - return std::accumulate(reference_tokens.begin(), reference_tokens.end(), - string_t{}, - [](const string_t& a, const string_t& b) + // one buffer: every token adds its escaped form (at least its length) and a slash + std::size_t length = 0; + for (const auto& reference_token : reference_tokens) { - return detail::concat(a, '/', detail::escape(b)); - }); + length += reference_token.size() + 1; + } + + string_t result; + result.reserve(length); + for (const auto& reference_token : reference_tokens) + { + result.append("/", 1); + detail::escape_into(result, reference_token); + } + return result; } /// @brief return a string representation of the JSON pointer @@ -129,21 +137,29 @@ class json_pointer friend json_pointer operator/(const json_pointer& lhs, const json_pointer& rhs) { - return json_pointer(lhs) /= rhs; + json_pointer result; + result.reference_tokens.reserve(lhs.reference_tokens.size() + rhs.reference_tokens.size()); + result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); + result.reference_tokens.insert(result.reference_tokens.end(), rhs.reference_tokens.begin(), rhs.reference_tokens.end()); + return result; } /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ friend json_pointer operator/(const json_pointer& lhs, string_t token) // NOLINT(performance-unnecessary-value-param) { - return json_pointer(lhs) /= std::move(token); + json_pointer result; + result.reference_tokens.reserve(lhs.reference_tokens.size() + 1); + result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); + result.reference_tokens.push_back(std::move(token)); + return result; } /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx) { - return json_pointer(lhs) /= array_idx; + return lhs / detail::to_string(array_idx); } /// @brief returns the parent of this JSON pointer @@ -155,8 +171,8 @@ class json_pointer return *this; } - json_pointer res = *this; - res.pop_back(); + json_pointer res; + res.reference_tokens = std::vector(reference_tokens.begin(), reference_tokens.end() - 1); return res; } @@ -245,14 +261,14 @@ class json_pointer private: /// @throw out_of_range.405 always - JSON_HEDLEY_NO_RETURN static void throw_no_parent() + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_no_parent() { JSON_THROW(detail::out_of_range::create(detail::exception_id::patch_on_root, "JSON pointer has no parent", nullptr)); } /// @throw out_of_range.404 always template - JSON_HEDLEY_NO_RETURN static void throw_unresolved(const string_t& reference_token, BasicJsonContext context) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_unresolved(const string_t& reference_token, BasicJsonContext context) { static_cast(reference_token); // unused when JSON_NOEXCEPTION is defined static_cast(context); @@ -305,6 +321,28 @@ class json_pointer return array_index_status::not_a_number; } + // fast path: up to 18 decimal digits cannot overflow unsigned long long or + // size_type (9 digits for a 32-bit size_type), so strtoull and errno are not needed + constexpr std::size_t max_fast_digits = sizeof(size_type) >= 8 ? 18 : 9; + if (!s.empty() && s.size() <= max_fast_digits) + { + size_type acc = 0; + for (std::size_t i = 0; i < s.size(); ++i) + { + const char c = s[i]; + if (JSON_HEDLEY_UNLIKELY(c < '0' || c > '9')) + { + break; // not a plain number: let strtoull decide + } + acc = static_cast(acc * 10 + static_cast(c - '0')); + if (JSON_HEDLEY_UNLIKELY(i == s.size() - 1)) + { + idx = acc; + return array_index_status::ok; + } + } + } + const char* p = s.data(); char* p_end = nullptr; // NOLINT(misc-const-correctness) errno = 0; // strtoull doesn't reset errno @@ -361,6 +399,83 @@ class json_pointer return idx; } + /*! + @brief compare with a JSON pointer string without splitting it into tokens + + @throw parse_error.107 if the string is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ + bool equals_string(const string_t& reference_string) const + { + if (reference_string.empty()) + { + return reference_tokens.empty(); + } + + if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/')) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr)); + } + + // the whole string is validated, whether or not it matches + const std::size_t size = reference_string.size(); + for (std::size_t pos = reference_string.find_first_of('~'); + pos != string_t::npos; + pos = reference_string.find_first_of('~', pos + 1)) + { + if (JSON_HEDLEY_UNLIKELY(pos == size - 1 || (reference_string[pos + 1] != '0' && reference_string[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + } + } + + // compare token by token, unescaping the string on the fly + std::size_t pos = 0; // the slash in front of the next token + for (const auto& reference_token : reference_tokens) + { + if (pos >= size) + { + return false; + } + + std::size_t i = pos + 1; + std::size_t j = 0; + const std::size_t token_size = reference_token.size(); + while (i < size && reference_string[i] != '/') + { + char c = reference_string[i++]; + if (c == '~') + { + c = (reference_string[i++] == '0') ? '~' : '/'; + } + if (j == token_size || reference_token[j] != c) + { + return false; + } + ++j; + } + if (j != token_size) + { + return false; + } + pos = i; + } + return pos == size; + } + + /// @brief object member access with a key that is only copied if it is inserted + template + static BasicJsonType& object_member(BasicJsonType& object, const string_t& key) + { + auto& members = *object.m_data.m_value.object; + const auto it = members.find(key); + if (it != members.end()) + { + return it->second; + } + return object.operator[](key); + } + JSON_PRIVATE_UNLESS_TESTED: json_pointer top() const { @@ -384,84 +499,141 @@ class json_pointer the result independent of the order in which the flattened object is iterated, which is unspecified for some object types. - The prefixes form a tree and are numbered, so each of them is stored only - once (as a node) rather than as a copy of all of its reference tokens. + The prefixes form a tree and are numbered (number 0 is the empty prefix), + so each of them is stored only once (as a node) rather than as a copy of + all of its reference tokens. The tree does not own the reference tokens: + it refers to the tokens of the parsed keys, which must outlive it. */ struct prefix_tree { - // children[id] maps a reference token to the number of the prefix - // extended by that token; number 0 is the empty prefix - std::vector> children; + // open-addressing hash table from (parent, last reference token) to + // the number of the node; empty slots have token == nullptr + struct slot + { + std::size_t hash; + std::size_t parent; + const string_t* token; + std::size_t id; + + slot() noexcept + : hash(0), parent(0), token(nullptr), id(0) + {} + + slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept + : hash(h), parent(p), token(t), id(i) + {} + }; + + std::vector slots; + std::size_t used = 0; + + static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept + { + std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL); + for (std::size_t i = 0; i < token.size(); ++i) + { + h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL); + } + return h ^ (h >> 29u); + } + + void grow() + { + std::vector old(slots.size() * 2); + old.swap(slots); + const std::size_t mask = slots.size() - 1; + for (const auto& s : old) + { + if (s.token != nullptr) + { + std::size_t i = s.hash & mask; + while (slots[i].token != nullptr) + { + i = (i + 1) & mask; + } + slots[i] = s; + } + } + } + + // parent[id] is the number of the prefix without its last token + std::vector parent; // is_array[id] is true iff some flattened key has the reference token // 0 directly below the prefix with number id std::vector is_array; // start with the empty prefix only prefix_tree() - : children(1) + : slots(16) + , parent(1, 0) , is_array(1, false) {} // return the number of the prefix with number id extended by // reference_token, adding it if it is new - std::size_t add_child(std::size_t id, string_t&& reference_token) + std::size_t add_child(std::size_t id, const string_t& reference_token) { + const std::size_t hash = hash_of(id, reference_token); + const std::size_t mask = slots.size() - 1; + std::size_t i = hash & mask; + while (slots[i].token != nullptr) + { + if (slots[i].hash == hash && slots[i].parent == id && *slots[i].token == reference_token) + { + return slots[i].id; + } + i = (i + 1) & mask; + } + + // new prefix + const std::size_t child = parent.size(); + slots[i] = slot(hash, id, &reference_token, child); if (reference_token == "0") { is_array[id] = true; } + parent.push_back(id); + is_array.push_back(false); - // read the number before the emplace_back below, which may - // reallocate children and invalidate the iterator - const std::size_t next = children.size(); - const auto inserted = children[id].emplace(std::move(reference_token), next); - const std::size_t child = inserted.first->second; - if (inserted.second) + // keep the load factor below 1/2 + if (++used * 2 > slots.size()) { - children.emplace_back(); - is_array.push_back(false); + grow(); } return child; } - - // return the number of the prefix with number id extended by - // reference_token, which must have been added before - std::size_t find_child(std::size_t id, const string_t& reference_token) const - { - const auto it = children[id].find(reference_token); - JSON_ASSERT(it != children[id].end()); - return it->second; - } }; /*! @brief create and return a reference to the pointed to value - Complexity: Linear in the number of reference tokens (times the logarithm - of the number of siblings for the prefix lookup). + @param[in] j the value to create the pointed to value in + @param[in] tokens the reference tokens of the pointer + @param[in] prefixes prefixes[i] is the number in @a tree of the prefix + consisting of the first i reference tokens + + Complexity: Linear in the number of reference tokens. @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ template - BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const + static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector& tokens, + const std::vector& prefixes, const prefix_tree& tree) { auto* result = &j; - // the number of the prefix consumed so far; used to look up whether - // the value to be created below is an array or an object - std::size_t id = 0; - // in case no reference tokens exist, return a reference to the JSON value // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) + for (std::size_t i = 0; i < tokens.size(); ++i) { + const auto& reference_token = tokens[i]; switch (result->type()) { case detail::value_t::null: { - if (tree.is_array[id]) + if (tree.is_array[prefixes[i]]) { // some reference token below this position is 0, so the // value is an array @@ -478,7 +650,7 @@ class json_pointer case detail::value_t::object: { // create an entry in the object - result = &result->operator[](reference_token); + result = &object_member(*result, reference_token); break; } @@ -505,8 +677,6 @@ class json_pointer default: JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j)); } - - id = tree.find_child(id, reference_token); } return *result; @@ -558,7 +728,7 @@ class json_pointer case detail::value_t::object: { // use unchecked object access - ptr = &ptr->operator[](reference_token); + ptr = &object_member(*ptr, reference_token); break; } @@ -608,8 +778,13 @@ class json_pointer { case detail::value_t::object: { - // note: at performs range check - ptr = &ptr->at(reference_token); + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) + { + // at() throws out_of_range.403 with the usual message + ptr->at(reference_token); + } + ptr = &it->second; break; } @@ -630,7 +805,7 @@ class json_pointer JSON_THROW(detail::out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat( "array index ", std::to_string(idx), " is out of range"), ptr)); } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -678,7 +853,9 @@ class json_pointer { // use unchecked object access; the const operator[] // asserts that the key exists - ptr = &ptr->operator[](reference_token); + const auto it = ptr->m_data.m_value.object->find(reference_token); + JSON_ASSERT(it != ptr->m_data.m_value.object->end()); + ptr = &it->second; break; } @@ -692,7 +869,9 @@ class json_pointer // use unchecked array access; the const operator[] // asserts that the index exists - ptr = &ptr->operator[](array_index(reference_token)); + const auto idx = array_index(reference_token); + JSON_ASSERT(idx < ptr->m_data.m_value.array->size()); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -733,12 +912,12 @@ class json_pointer { case detail::value_t::object: { - const auto it = ptr->find(reference_token); - if (JSON_HEDLEY_UNLIKELY(it == ptr->end())) + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) { return nullptr; } - ptr = &*it; + ptr = &it->second; break; } @@ -775,7 +954,7 @@ class json_pointer { return nullptr; } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -808,13 +987,14 @@ class json_pointer { case detail::value_t::object: { - if (!ptr->contains(reference_token)) + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (it == ptr->m_data.m_value.object->end()) { // we did not find the key in the object return false; } - ptr = &ptr->operator[](reference_token); + ptr = &it->second; break; } @@ -836,13 +1016,13 @@ class json_pointer return false; } - if (idx >= ptr->size()) + if (idx >= ptr->m_data.m_value.array->size()) { // index out of range return false; } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -892,6 +1072,15 @@ class json_pointer JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr)); } + // every slash starts a reference token + const std::size_t size = reference_string.size(); + std::size_t token_count = 0; + for (std::size_t i = 0; i < size; ++i) + { + token_count += (reference_string[i] == '/') ? 1 : 0; + } + result.reserve(token_count); + // extract the reference tokens: // - slash: position of the last read slash (or end of string) // - start: position after the previous slash @@ -913,31 +1102,52 @@ class json_pointer const auto count = (slash == string_t::npos ? reference_string.size() : slash) - start; auto reference_token = string_t(reference_string.data() + start, count); - // check reference tokens are properly escaped - for (std::size_t pos = reference_token.find_first_of('~'); - pos != string_t::npos; - pos = reference_token.find_first_of('~', pos + 1)) + // check reference tokens are properly escaped; a token without + // a '~' (the common case) needs neither check nor unescaping + std::size_t pos = reference_token.find_first_of('~'); + if (pos != string_t::npos) { - JSON_ASSERT(reference_token[pos] == '~'); - - // ~ must be followed by 0 or 1 - if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || - (reference_token[pos + 1] != '0' && - reference_token[pos + 1] != '1'))) + for (; pos != string_t::npos; pos = reference_token.find_first_of('~', pos + 1)) { - JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + JSON_ASSERT(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || + (reference_token[pos + 1] != '0' && + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + } } + + detail::unescape(reference_token); } // finally, store the reference token - detail::unescape(reference_token); - result.push_back(reference_token); + result.push_back(std::move(reference_token)); } return result; } private: + // Add the entry (key, v) to the object `obj`. Flattened paths are pairwise + // distinct, so objects that can append a key known to be absent + // (ordered_map) skip the lookup, which is linear and would make flatten + // quadratic; all other object types use operator[]. + template + static auto add_flattened(ObjectType& obj, const string_t& key, ValueType&& v, detail::priority_tag<1> /*unused*/) + -> decltype(obj.emplace_new_key(key, std::forward(v)), void()) + { + obj.emplace_new_key(key, std::forward(v)); + } + + template + static void add_flattened(ObjectType& obj, const string_t& key, ValueType&& v, detail::priority_tag<0> /*unused*/) + { + obj[key] = std::forward(v); + } + /*! @param[in] reference_string the reference string to the current value @param[in] value the value to consider @@ -982,7 +1192,8 @@ class json_pointer // flatten `v`, whose path is `path`: primitives and empty containers // are added to the result right away; other containers get a frame - const auto enter = [&stack, &path, &result](const BasicJsonType & v) + auto& object = *result.m_data.m_value.object; + const auto enter = [&stack, &path, &object](const BasicJsonType & v) { switch (v.type()) { @@ -991,7 +1202,7 @@ class json_pointer if (v.m_data.m_value.array->empty()) { // flatten empty array as null - result[path] = nullptr; + add_flattened(object, path, nullptr, detail::priority_tag<1> {}); } else { @@ -1005,7 +1216,7 @@ class json_pointer if (v.m_data.m_value.object->empty()) { // flatten empty object as null - result[path] = nullptr; + add_flattened(object, path, nullptr, detail::priority_tag<1> {}); } else { @@ -1025,7 +1236,7 @@ class json_pointer default: { // add a primitive value with its reference string - result[path] = v; + add_flattened(object, path, v, detail::priority_tag<1> {}); return; } } @@ -1071,6 +1282,10 @@ class json_pointer enter(it->second); } } + + // the entries were added without going through basic_json, so the + // parent pointers (JSON_DIAGNOSTICS) have to be set now + result.set_parents(); } /*! @@ -1094,21 +1309,53 @@ class json_pointer BasicJsonType result; + // parse every key once; the reference tokens are kept for the second + // pass (the vectors are never resized, so the tree may refer to them) + std::vector> keys; + keys.reserve(value.m_data.m_value.object->size()); + for (const auto& element : *value.m_data.m_value.object) + { + json_pointer ptr(element.first); + keys.push_back(std::move(ptr.reference_tokens)); + } + // collect the pointer prefixes that have a reference token 0 among // their children; the values below them are arrays, all others are // objects (see prefix_tree) prefix_tree tree; - for (const auto& element : *value.m_data.m_value.object) + // the number of the prefix with i + 1 tokens of the previous key; keys + // are often sorted, so most of the prefix is the same as for the + // previous key and need not be looked up again + std::vector path; + const std::vector* previous = nullptr; + // the number of the complete prefix (i.e., the key) of each key + std::vector leaves; + leaves.reserve(keys.size()); + for (const auto& tokens : keys) { - json_pointer ptr(element.first); - std::size_t id = 0; - for (auto& reference_token : ptr.reference_tokens) + std::size_t common = 0; + if (previous != nullptr) { - id = tree.add_child(id, std::move(reference_token)); + const std::size_t limit = (std::min)(tokens.size(), previous->size()); + while (common < limit && tokens[common] == (*previous)[common]) + { + ++common; + } } + path.resize(common); + std::size_t id = common == 0 ? 0 : path[common - 1]; + for (std::size_t i = common; i < tokens.size(); ++i) + { + id = tree.add_child(id, tokens[i]); + path.push_back(id); + } + leaves.push_back(id); + previous = &tokens; } // iterate the JSON object values + std::vector prefixes; + std::size_t key_index = 0; for (const auto& element : *value.m_data.m_value.object) { if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) @@ -1116,11 +1363,21 @@ class json_pointer JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second)); } + const auto& tokens = keys[key_index]; + // number the prefixes of this key from its last one to the first + prefixes.resize(tokens.size()); + for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i) + { + id = tree.parent[id]; + prefixes[i - 1] = id; + } + ++key_index; + // Assign the value to the reference pointed to by JSON pointer. Note // that if the JSON pointer is "" (i.e., points to the whole value), // function get_and_create returns a reference to the result itself. // An assignment will then create a primitive value. - json_pointer(element.first).get_and_create(result, tree) = element.second; + get_and_create(result, tokens, prefixes, tree) = element.second; } return result; @@ -1159,7 +1416,7 @@ class json_pointer = delete; #else { - return *this == json_pointer(rhs); + return equals_string(rhs); } #endif @@ -1243,7 +1500,7 @@ inline bool operator==(const json_pointer& lhs, = delete; #else { - return lhs == json_pointer(rhs); + return lhs.equals_string(rhs); } #endif @@ -1256,7 +1513,7 @@ inline bool operator==(const StringType& lhs, = delete; #else { - return json_pointer(lhs) == rhs; + return rhs.equals_string(lhs); } #endif diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index 232a6db51..2f855f307 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -22,6 +22,8 @@ #undef JSON_INTERNAL_ALWAYS_INLINE #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION +#undef JSON_DTOA_SSE2 +#undef JSON_DTOA_NEON #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH diff --git a/include/nlohmann/detail/meta/type_traits.hpp b/include/nlohmann/detail/meta/type_traits.hpp index a08ad1500..397bb6287 100644 --- a/include/nlohmann/detail/meta/type_traits.hpp +++ b/include/nlohmann/detail/meta/type_traits.hpp @@ -9,9 +9,11 @@ #pragma once #include // numeric_limits +#include // map, multimap #include // char_traits #include // tuple #include // false_type, is_constructible, is_integral, is_same, true_type +#include // unordered_map, unordered_multimap #include // declval #include // vector #if defined(__cpp_lib_byte) && __cpp_lib_byte >= 201603L @@ -683,6 +685,22 @@ struct is_compatible_integer_type : is_compatible_integer_type_impl {}; +// whether to_json() stores an integer of type CompatibleNumberIntegerType as +// number_integer +template +struct is_compatible_number_integer_type + : is_compatible_integer_type {}; + +// if number_unsigned_t is the same type as number_integer_t, unsigned integers +// are stored as number_integer as well (#2573) +template +struct is_compatible_number_integer_type, + std::is_integral, + std::is_unsigned, + negation>>::value>> + : std::is_constructible {}; + template struct is_compatible_type_impl: std::false_type {}; @@ -897,6 +915,31 @@ struct has_capacity : std::integral_constant struct is_ordered_map : has_capacity {}; +// whether T is a specialization of nlohmann::ordered_map, which only ever +// appends members, so they can only move when its capacity changes +template +struct is_nlohmann_ordered_map : std::false_type {}; + +template +struct is_nlohmann_ordered_map> : std::true_type {}; + +// whether adding or erasing elements never moves the other elements of the +// container T; only node-based standard containers are known to guarantee this +template +struct has_stable_elements : std::false_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + // to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > T conditional_static_cast(U value) @@ -1020,6 +1063,25 @@ constexpr bool value_in_range_of(T val) template using bool_constant = std::integral_constant; +// Stands in for number_unsigned_t in overload sets if it is the same type as +// number_integer_t, so that the unsigned overloads do not collide with the +// integer ones. Never constructed. Such a basic_json stores every integer as +// number_integer (#2573). +struct same_as_number_integer_type {}; + +// NumberUnsignedType, or same_as_number_integer_type if it is the same type as +// NumberIntegerType; basic_json uses this one, as it cannot name its own member +// types through an incomplete basic_json (MSVC reports a recursive alias, and +// old GCC versions crash) +template +using number_unsigned_overload_type = typename std::conditional::value, + same_as_number_integer_type, NumberUnsignedType>::type; + +// number_unsigned_t of BasicJsonType, or same_as_number_integer_type if it is +// the same type as number_integer_t +template +using number_unsigned_overload_t = number_unsigned_overload_type; + /////////////////////////////////////////////////////////////////////////////// // is_c_string /////////////////////////////////////////////////////////////////////////////// diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 2a18ea600..3a9abbc08 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -221,7 +221,7 @@ class binary_writer case value_t::number_unsigned: { - write_cbor_head(0x00, j.m_data.m_value.number_unsigned); + write_cbor_head(0x00, conditional_static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -1601,7 +1601,7 @@ class binary_writer return calc_bson_integer_size(j.m_data.m_value.number_integer); case value_t::number_unsigned: - return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); + return calc_bson_unsigned_size(conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return calc_bson_string_size(*j.m_data.m_value.string, j); @@ -1643,7 +1643,7 @@ class binary_writer return write_bson_integer(name, j.m_data.m_value.number_integer); case value_t::number_unsigned: - return write_bson_unsigned(name, j.m_data.m_value.number_unsigned); + return write_bson_unsigned(name, conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return write_bson_string(name, *j.m_data.m_value.string); diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 64c4fcf03..3621a78d9 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -49,6 +49,12 @@ class serializer using number_float_t = typename BasicJsonType::number_float_t; using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type of the absolute value of an integer: number_unsigned_t, or its + /// unsigned counterpart if it is signed (it may be number_integer_t, #2573) + using number_magnitude_t = typename std::conditional < + std::is_integral::value && std::is_signed::value, + std::make_unsigned, + std::enable_if>::type::type; using binary_char_t = typename BasicJsonType::binary_t::value_type; public: @@ -772,6 +778,33 @@ class serializer break; } } + + // A byte below 0x60 is a complete code point and needs no + // UTF-8 decoding. The scanner stops only at a control + // character, a quote, or a backslash in that range, so look up + // its escape sequence and write all 8 bytes of the table entry + // straight into the write buffer; only the length stored in the + // last byte of the entry is kept. + if (data[i] < 0x60) + { + // emit any bytes still pending in string_buffer first to + // preserve output order + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + const auto& escape = ascii_escapes()[data[i]]; + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size()); + write_buffer_pos += static_cast(escape[7]); + bytes_after_last_accept = 0; + undumped_chars = 0; + continue; + } } const auto byte = static_cast(s[i]); @@ -780,92 +813,39 @@ class serializer { case UTF8_ACCEPT: // decode found a new code point { - switch (codepoint) + // the code point is at least 0x60 (smaller ones are handled + // above) and is not printable ASCII (which the scanner + // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII + // character + if (EnsureAscii) { - case 0x08: // backspace + JSON_ASSERT(codepoint >= 0x7F); + if (error_handler == error_handler_t::keep) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; + // this character was buffered as raw bytes below + // in case it turned out to be part of an + // ill-formed sequence (which is kept as is); now + // that it decoded to a well-formed code point, + // undo that and \u-escape it like any other + // character instead + bytes = bytes_after_last_accept; } - - case 0x09: // horizontal tab + if (codepoint <= 0xFFFF) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; + write_u_escape(bytes, static_cast(codepoint)); } - - case 0x0A: // newline + else { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // EnsureAscii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) - { - if (EnsureAscii && error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes - // below in case it turned out to be part of - // an ill-formed sequence (which is kept as - // is); now that it decoded to a well-formed - // code point, undo that and \u-escape it - // like any other character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; + write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); + write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); } } + else + { + // copy byte to buffer (all previous bytes of the + // sequence have been copied in the default case below) + string_buffer[bytes++] = s[i]; + } // write buffer and reset index; there must be 13 bytes // left, as this is the maximal number of bytes to be @@ -1227,7 +1207,7 @@ class serializer @param[in] x unsigned integer number to count its digits @return number of decimal digits */ - unsigned int count_digits(number_unsigned_t x) noexcept + unsigned int count_digits(number_magnitude_t x) noexcept { unsigned int n_digits = 1; for (;;) @@ -1253,6 +1233,121 @@ class serializer } } + /*! + @brief the JSON form of each character below 0x60 + + Entry c holds the escape sequence of character c - the short form for + \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters - + or c itself if it needs no escaping, padded with zeros to 7 bytes. The last + byte is the length of the sequence. The 8 bytes of an entry are copied at + once, so writing an escape needs neither a branch nor a byte-wise loop. + */ + static const std::array, 0x60>& ascii_escapes() noexcept + { + static const std::array, 0x60> escapes = + { + { + {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00 + {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01 + {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02 + {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03 + {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04 + {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05 + {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06 + {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07 + {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08 + {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09 + {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A + {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B + {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C + {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D + {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E + {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F + {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10 + {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11 + {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12 + {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13 + {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14 + {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15 + {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16 + {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17 + {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18 + {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19 + {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A + {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B + {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C + {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D + {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E + {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F + {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20 + {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21 + {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22 + {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23 + {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24 + {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25 + {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26 + {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27 + {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28 + {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29 + {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A + {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B + {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C + {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D + {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E + {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F + {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30 + {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31 + {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32 + {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33 + {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34 + {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35 + {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36 + {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37 + {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38 + {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39 + {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A + {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B + {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C + {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D + {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E + {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F + {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40 + {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41 + {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42 + {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43 + {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44 + {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45 + {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46 + {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47 + {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48 + {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49 + {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A + {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B + {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C + {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D + {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E + {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F + {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50 + {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51 + {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52 + {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53 + {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54 + {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55 + {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56 + {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57 + {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58 + {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59 + {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A + {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B + {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C + {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D + {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E + {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F + } + }; + return escapes; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * @@ -1366,8 +1461,9 @@ class serializer /*! @brief dump an integer - Dump a given integer, appending it to @ref write_buffer. Works internally with - @a number_buffer. + Dump a given integer, appending it to @ref write_buffer (directly: copying + the digits from another buffer right after writing them waits until the + stores are done). @param[in] x integer number (signed or unsigned) to dump @tparam NumberType either @a number_integer_t or @a number_unsigned_t @@ -1402,33 +1498,57 @@ class serializer return; } - // use a pointer to fill the buffer - auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto) + // use a pointer to fill the buffer (room for as much as number_buffer holds) + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + number_buffer.size() > write_buffer.size())) + { + flush(); + } + auto* buffer_ptr = write_buffer.data() + write_buffer_pos; - number_unsigned_t abs_value; + number_magnitude_t abs_value; - unsigned int n_chars{}; + // one byte for the minus sign + unsigned int n_chars = 0; if (is_negative_number(x)) { *buffer_ptr = '-'; abs_value = remove_sign(static_cast(x)); - - // account one more byte for the minus sign - n_chars = 1 + count_digits(abs_value); + n_chars = 1; } else { - abs_value = static_cast(x); - n_chars = count_digits(abs_value); + abs_value = static_cast(x); } + // up to 16 digits: eight at a time (as the digits of floats), written + // without leading zeros + if (abs_value < 10000000000000000u) + { + const std::uint64_t value = abs_value; + const std::uint64_t upper = value / 100000000u; + const std::uint64_t first = dtoa_impl::eight_digit_bytes(upper != 0 ? upper : value); + const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) + char* const p = buffer_ptr + n_chars; + dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u); + n_chars += 8 - leading; + if (upper != 0) + { + dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(value - (upper * 100000000u)) + 0x3030303030303030u); + n_chars += 8; + } + write_buffer_pos += n_chars; + return; + } + + n_chars += count_digits(abs_value); + // spare 1 byte for '\0' JSON_ASSERT(n_chars < number_buffer.size() - 1); // jump to the end to generate the string from backward, // so we later avoid reversing the result - buffer_ptr += static_cast(n_chars); + buffer_ptr += n_chars; // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu // See: https://www.youtube.com/watch?v=o4-CwDo2zpg @@ -1451,14 +1571,13 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - put_buffer(number_buffer, n_chars); + write_buffer_pos += n_chars; } /*! @brief dump a floating-point number - Dump a given floating-point number, appending it to @ref write_buffer. Works internally - with @a number_buffer. + Dump a given floating-point number, appending it to @ref write_buffer. @param[in] x floating-point number to dump */ @@ -1485,10 +1604,15 @@ class serializer void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) { - auto* begin = number_buffer.data(); + // directly into the write buffer: copying the text from number_buffer + // right after to_chars() wrote it waits until its stores are done + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + number_buffer.size() > write_buffer.size())) + { + flush(); + } + auto* begin = write_buffer.data() + write_buffer_pos; auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - - put_buffer(number_buffer, static_cast(end - begin)); + write_buffer_pos += static_cast(end - begin); } JSON_HEDLEY_NON_NULL(1) @@ -1562,17 +1686,6 @@ class serializer } } - /* - * Overload to make the compiler happy while it is instantiating - * dump_integer for number_unsigned_t. - * Must never be called. - */ - number_unsigned_t remove_sign(number_unsigned_t x) - { - JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE - return x; // LCOV_EXCL_LINE - } - /* * Helper function for dump_integer * @@ -1582,10 +1695,10 @@ class serializer * absolute values of INT_MIN and INT_MAX are usually not the same. See * #1708 for details. */ - number_unsigned_t remove_sign(number_integer_t x) noexcept + number_magnitude_t remove_sign(number_integer_t x) noexcept { JSON_ASSERT(x < 0); - return static_cast(-(x + 1)) + 1; + return static_cast(-(x + 1)) + 1; } private: diff --git a/include/nlohmann/detail/string_escape.hpp b/include/nlohmann/detail/string_escape.hpp index a6489e054..b3308b5c3 100644 --- a/include/nlohmann/detail/string_escape.hpp +++ b/include/nlohmann/detail/string_escape.hpp @@ -9,6 +9,7 @@ #pragma once #include // size_t +#include // move #include @@ -24,39 +25,46 @@ namespace detail * Note the order of escaping "~" to "~0" and "/" to "~1" is important. * * The string is rebuilt in a single pass, appending whole runs between the - * characters that need escaping. Scanning with find_first_of() keeps the - * common case -- nothing to escape -- as fast as a single search, while - * repeated replace() calls would move the tail of the string once per - * escaped character. + * characters that need escaping, while repeated replace() calls would move + * the tail of the string once per escaped character. @ref escape_into appends + * the escaped string to an existing one. */ +template +inline void escape_into(StringType& out, const StringType& s) +{ + const std::size_t size = s.size(); + std::size_t run = 0; + for (std::size_t i = 0; i < size; ++i) + { + const char c = s[i]; + if (c == '~' || c == '/') + { + out.append(s.data() + run, i - run); + out.append(c == '~' ? "~0" : "~1", 2); + run = i + 1; + } + } + out.append(s.data() + run, size - run); +} + template inline StringType escape(const StringType& s) { - auto next_special = [&s](std::size_t from) + // a single scan for both special characters; nothing to escape is the common case + const std::size_t size = s.size(); + std::size_t i = 0; + while (i < size && s[i] != '~' && s[i] != '/') { - const auto tilde = s.find_first_of('~', from); - const auto slash = s.find_first_of('/', from); - return tilde < slash ? tilde : slash; // npos is the largest value - }; - - auto pos = next_special(0); - if (pos == StringType::npos) + ++i; + } + if (i == size) { return s; } StringType result; - result.reserve(s.size() + 2); - - std::size_t run = 0; - while (pos != StringType::npos) - { - result.append(s.data() + run, pos - run); - result.append(s[pos] == '~' ? "~0" : "~1", 2); - run = pos + 1; - pos = next_special(run); - } - result.append(s.data() + run, s.size() - run); + result.reserve(size + 2); + escape_into(result, s); return result; } @@ -101,7 +109,7 @@ inline void unescape(StringType& s) pos = s.find_first_of('~', run); } result.append(s.data() + run, s.size() - run); - s = result; + s = std::move(result); } } // namespace detail diff --git a/include/nlohmann/detail/value_t.hpp b/include/nlohmann/detail/value_t.hpp index 9fc217bcc..cda0facd2 100644 --- a/include/nlohmann/detail/value_t.hpp +++ b/include/nlohmann/detail/value_t.hpp @@ -117,6 +117,34 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept } #endif +/*! +@brief whether a value of the given type owns heap memory + +Strings, binaries, objects, and arrays are stored as pointers to allocated +values; all other types are stored directly in the union. Every enumerator is +listed so that -Wswitch-enum flags this function when a new type is added. +*/ +inline bool owns_memory(const value_t t) noexcept +{ + switch (t) + { + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::binary: + return true; + + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + return false; + } +} + /*! @brief compare an integer with a floating point number without precision loss diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 3cdb2d65a..ea74a29da 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -42,9 +42,14 @@ // each of these, not only the forward declarations from json_fwd.hpp, so IWYU's suggestion to // drop them (nothing in this file otherwise names the type) would break every downstream // translation unit that relies on basic_json<>'s defaults actually being usable. -#include // IWYU pragma: keep +// export: client code names what these headers declare (nlohmann::json and ordered_json from +// json_fwd.hpp, adl_serializer for specializations, json_ref behind brace initialization, the +// NLOHMANN_DEFINE_* and NLOHMANN_JSON_NAMESPACE_* macros, and the literals at the end of this file), and include analyzers must +// attribute it to this header, not tell clients to include the internal ones (issue #5812). +// tests/iwyu_consumer checks this. +#include // IWYU pragma: export #include -#include +#include // IWYU pragma: export #include // IWYU pragma: keep #include // IWYU pragma: keep #include @@ -61,8 +66,8 @@ #include #include #include -#include -#include +#include // IWYU pragma: export +#include // IWYU pragma: export #include #include #include @@ -74,7 +79,7 @@ #include #include #include -#include +#include // IWYU pragma: export #include // IWYU pragma: keep #include @@ -495,8 +500,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec "NumberUnsignedType must be at least as wide as NumberIntegerType, " "because it has to hold the absolute value of every NumberIntegerType value"); + // NumberUnsignedType may be the same type as NumberIntegerType (#2573), but + // a repeated NumberFloatType would give overloads that collide, so it gets + // a readable diagnostic instead. + static_assert(!std::is_same::value && !std::is_same::value, + "NumberFloatType must be a different type than NumberIntegerType and NumberUnsignedType"); + private: + /// the kind a value of kind @a t is stored as: if number_unsigned_t is the + /// same type as number_integer_t, every integer is stored as number_integer + /// (#2573); every other kind is stored as itself + static constexpr value_t stored_kind(const value_t t) noexcept + { + return (std::is_same::value && t == value_t::number_unsigned) ? value_t::number_integer : t; + } + /// helper for exception-safe object creation template JSON_HEDLEY_RETURNS_NON_NULL @@ -570,8 +589,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec json_value(boolean_t v) noexcept : boolean(v) {} /// constructor for numbers (integer) json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (unsigned); takes a placeholder if number_unsigned_t is number_integer_t + json_value(detail::number_unsigned_overload_type v) noexcept : number_unsigned(v) {} /// constructor for numbers (floating-point) json_value(number_float_t v) noexcept : number_float(v) {} /// constructor for empty values of a given type @@ -580,26 +599,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec switch (t) { case value_t::object: - { - object = create(); - break; - } - case value_t::array: - { - array = create(); - break; - } - case value_t::string: - { - string = create(""); - break; - } - case value_t::binary: { - binary = create(); + // allocating: kept out of line, so the scalar cases below inline + init_owned(t); break; } @@ -639,13 +644,41 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec object = nullptr; // silence warning, see #821 if (JSON_HEDLEY_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(detail::exception_id::internal_error, "961c151d2e87f2686a955a9be24d316f1362bf21 3.12.0", nullptr)); // LCOV_EXCL_LINE + detail::throw_literal(detail::exception_id::internal_error, "961c151d2e87f2686a955a9be24d316f1362bf21 3.12.0", nullptr); // LCOV_EXCL_LINE } break; } } } + JSON_HEDLEY_NEVER_INLINE + void init_owned(value_t t) + { + switch (t) + { + case value_t::object: + object = create(); + break; + case value_t::array: + array = create(); + break; + case value_t::string: + string = create(""); + break; + case value_t::binary: + binary = create(); + break; + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + break; + } + } + /// constructor for strings json_value(const string_t& value) : string(create(value)) {} @@ -715,6 +748,34 @@ private: } } + // true if no child of the non-empty array/object v has children of + // its own, i.e. v can be freed in one go (its destructor then + // destroys scalars and empty containers only, so it recurses at most + // one level deep); stops at the first structured child it meets + static bool all_children_leaves(const basic_json& v) noexcept + { + if (v.m_data.m_type == value_t::array) + { + const auto& a = *v.m_data.m_value.array; + for (auto it = a.rbegin(); it != a.rend(); ++it) + { + if (!has_no_children(*it)) + { + return false; + } + } + return true; + } + for (const auto& el : *v.m_data.m_value.object) + { + if (!has_no_children(el.second)) + { + return false; + } + } + return true; + } + // The walk in destroy_container() may take the children of a // container in any order, as long as it picks the same child again // while that container is not modified in between. Arrays and @@ -770,15 +831,16 @@ private: } } - // deallocates the (already empty) array/object held by v; this is - // the same allocator-based free the old recursive implementation - // used, just factored out so every level of the walk in destroy() - // can share it + // deallocates the array/object held by v, which is either empty or + // holds only scalars and empty containers (so its destructor + // recurses at most one level deep); this is the same allocator-based + // free the old recursive implementation used, just factored out so + // every level of the walk in destroy() can share it static void free_container(basic_json& v) noexcept { + JSON_ASSERT(has_no_children(v) || all_children_leaves(v)); if (v.m_data.m_type == value_t::array) { - JSON_ASSERT(v.m_data.m_value.array->empty()); AllocatorType alloc; std::allocator_traits::destroy(alloc, v.m_data.m_value.array); std::allocator_traits::deallocate(alloc, v.m_data.m_value.array, 1); @@ -786,7 +848,6 @@ private: else { JSON_ASSERT(v.m_data.m_type == value_t::object); - JSON_ASSERT(v.m_data.m_value.object->empty()); AllocatorType alloc; std::allocator_traits::destroy(alloc, v.m_data.m_value.object); std::allocator_traits::deallocate(alloc, v.m_data.m_value.object, 1); @@ -831,7 +892,58 @@ public: return; } - // Destroy the tree without recursing per nesting level and + // This json_value is not itself a basic_json, so the top-level + // container is first moved into a local stand-in ("cur"); a + // default-constructed basic_json has a null pointer in its + // m_value (see data::m_value's initializer), so swapping it + // with *this leaves this union's own pointer null, and it is + // never looked at or freed a second time. + basic_json cur; + cur.m_data.m_type = t; + using std::swap; + swap(cur.m_data.m_value, *this); + + destroy_bounded(cur, 0); + } + + // Destroy the array/object v, whose children are destroyed first. + // The first recursion_depth_limit() levels are destroyed by plain + // recursion, which is the fastest way and cannot throw; anything + // nested deeper is handed to destroy_walk(), which uses no call stack. + static void destroy_bounded(basic_json& v, std::size_t depth) noexcept + { + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + destroy_walk(v); + return; + } + + if (v.m_data.m_type == value_t::array) + { + for (auto& child : *v.m_data.m_value.array) + { + if (child.m_data.m_type == value_t::array || child.m_data.m_type == value_t::object) + { + destroy_bounded(child, depth + 1); + } + } + } + else + { + for (auto& child : *v.m_data.m_value.object) + { + if (child.second.m_data.m_type == value_t::array || child.second.m_data.m_type == value_t::object) + { + destroy_bounded(child.second, depth + 1); + } + } + } + + // only leaves and already destroyed (null) children remain + free_container(v); + } + + // Destroy the tree below top without recursing per nesting level and // without any heap allocation: a heap-allocated flattening // stack (the previous implementation) can itself throw // bad_alloc, which would escape this noexcept destructor and @@ -855,16 +967,24 @@ public: // pointer in its m_value (see data::m_value's initializer), // so swapping it with *this leaves this union's own pointer // null, and it is never looked at or freed a second time. + static void destroy_walk(basic_json& top) noexcept + { basic_json cur; - cur.m_data.m_type = t; - using std::swap; - swap(cur.m_data.m_value, *this); + take(cur, top); basic_json prev; // value_t::null: no parent + // true right after cur was entered: its children have not been + // looked at yet. If none of them has children of its own, cur is + // freed in one go instead of child by child; this is tried once + // per container, so the walk stays linear. + bool fresh = true; + while (true) { - if (has_no_children(cur)) + const bool fresh_now = fresh; + fresh = false; + if (has_no_children(cur) || (fresh_now && all_children_leaves(cur))) { if (prev.m_data.m_type == value_t::null) { @@ -903,10 +1023,22 @@ public: take(cur_child_ref, prev); take(prev, cur); take(cur, tmp); + fresh = true; } } + // keep the check whether t owns memory inline so destroying a scalar + // is a compare and a branch, and leave the actual work to an + // out-of-line function void destroy(value_t t) + { + if (JSON_HEDLEY_UNLIKELY(detail::owns_memory(t))) + { + destroy_owned(t); + } + } + + JSON_HEDLEY_NEVER_INLINE void destroy_owned(value_t t) { switch (t) { @@ -1074,11 +1206,16 @@ public: return next; } - /// @brief the capacity of the stored array, or unknown_size() - /// Only JSON_DIAGNOSTICS uses the value, to detect a reallocation that - /// would invalidate the parent pointers. Array types that do not have a - /// capacity() member function report unknown_size(), which is treated as - /// "the elements may have moved". + /// @brief the capacity of the stored array or object, or unknown_size() + /// Only JSON_DIAGNOSTICS uses the value, to detect an insertion that may + /// have moved the other elements and so invalidated their parent pointers. + /// unknown_size() means "the elements may have moved". Arrays only grow at + /// the end, so their capacity tells whether they reallocated; arrays + /// without a capacity() member report unknown_size(). For objects, only + /// types known to keep their elements in place (std::map and the other + /// node-based standard containers) report a fixed 0, and only + /// nlohmann::ordered_map, which appends members, reports its capacity; any + /// other object type (a sorted flat map, say) reports unknown_size(). #if JSON_DIAGNOSTICS template < typename A = array_t, detail::enable_if_t < detail::has_capacity::value, int > = 0 > std::size_t array_capacity() const noexcept @@ -1091,57 +1228,63 @@ public: { return detail::unknown_size(); } + + template < typename O = object_t, detail::enable_if_t < detail::has_stable_elements::value, int > = 0 > + static constexpr std::size_t object_capacity() noexcept + { + return 0; + } + + template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && detail::is_nlohmann_ordered_map::value, int > = 0 > + std::size_t object_capacity() const noexcept + { + return m_data.m_value.object->capacity(); + } + + template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && !detail::is_nlohmann_ordered_map::value, int > = 0 > + static constexpr std::size_t object_capacity() noexcept + { + return detail::unknown_size(); + } + + std::size_t container_capacity() const noexcept + { + return is_array() ? array_capacity() : object_capacity(); + } #else - static constexpr std::size_t array_capacity() noexcept + static constexpr std::size_t container_capacity() noexcept { return detail::unknown_size(); } #endif - /// @brief set the parent of a value that has just been added to an array - /// @param j the added value - /// @param old_capacity the value @ref array_capacity() returned before the + /// @brief set the parent of a value that has just been added to an array or object + /// @param j the added (or looked-up) value + /// @param old_capacity the value @ref container_capacity() returned before the /// insertion - reference set_parent_after_array_insert(reference j, std::size_t old_capacity) + reference set_parent_after_insert(reference j, std::size_t old_capacity) { #if JSON_DIAGNOSTICS // see https://github.com/nlohmann/json/issues/2838 - JSON_ASSERT(type() == value_t::array); if (JSON_HEDLEY_UNLIKELY(old_capacity == detail::unknown_size() - || array_capacity() != old_capacity)) + || container_capacity() != old_capacity)) { - // the capacity has changed, or the array type does not let us tell: - // the elements may have moved, so update all parents + // the capacity has changed, or the container type does not let us + // tell: the elements may have moved, so update all parents set_parents(); return j; } #else static_cast(old_capacity); #endif - return set_parent(j); + return set_parent_at(j); } - reference set_parent(reference j) + /// @brief set the parent of an element accessed via at(); never moves elements + reference set_parent_at(reference j) noexcept { #if JSON_DIAGNOSTICS - // ordered_json uses a vector internally, so pointers could have - // been invalidated; see https://github.com/nlohmann/json/issues/2962 -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - set_parents(); - return j; - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif - j.m_parent = this; -#else - static_cast(j); #endif return j; } @@ -2131,9 +2274,11 @@ public: /// @brief restore the parent pointers after erasing from an object - /// ordered_json keeps its members in a vector, and erasing a member - /// re-constructs every member after it in place, which resets their - /// parent pointers + /// Unless the object type is known to keep its elements in place (see + /// detail::has_stable_elements), erasing a member may move the members + /// after it; ordered_json, for instance, keeps its members in a vector + /// and re-constructs every member after the erased one in place, which + /// resets their parent pointers void set_parents_after_object_erase() { #if JSON_DIAGNOSTICS @@ -2141,7 +2286,7 @@ public: #pragma warning(push ) #pragma warning(disable : 4127) // ignore warning to replace if with if constexpr #endif - if (detail::is_ordered_map::value) + if (!detail::has_stable_elements::value) { set_parents(); } @@ -2176,7 +2321,7 @@ public: /// @brief create an empty value with a given type /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(const value_t v) - : m_data(v) + : m_data(stored_kind(v)) { assert_invariant(); } @@ -2257,7 +2402,7 @@ public: { convert_leaf(val); } - JSON_ASSERT(m_data.m_type == val.type()); + JSON_ASSERT(m_data.m_type == stored_kind(val.type())); set_parents(); assert_invariant(); @@ -2305,7 +2450,7 @@ public: // if an object is wanted but impossible, throw an exception if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object)) { - JSON_THROW(type_error::create(detail::exception_id::object_from_non_pairs, "cannot create object from initializer list", nullptr)); + detail::throw_literal(detail::exception_id::object_from_non_pairs, "cannot create object from initializer list", nullptr); } } @@ -2425,7 +2570,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterators_incompatible, "iterators are not compatible", nullptr)); + detail::throw_literal(detail::exception_id::iterators_incompatible, "iterators are not compatible", nullptr); } // copy type from the first iterator @@ -2444,7 +2589,7 @@ public: if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() || !last.m_it.primitive_iterator.is_end())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_out_of_range, "iterators out of range", first.m_object)); + detail::throw_literal(detail::exception_id::iterator_range_out_of_range, "iterators out of range", first.m_object); } break; } @@ -2512,7 +2657,7 @@ public: case value_t::null: case value_t::discarded: default: - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", first.m_object->type_name()), first.m_object)); + throw_iterator_range_of_null(first.m_object); } set_parents(); @@ -2541,18 +2686,16 @@ public: // check of passed value is valid other.assert_invariant(); - if (m_data.m_type == value_t::object || m_data.m_type == value_t::array) + if (JSON_HEDLEY_LIKELY(!detail::owns_memory(m_data.m_type))) { - // copying the container directly would call this constructor again - // for every element, once per nesting level - copy_structured(other); + // null, boolean, numbers, discarded: plain copy of the union + m_data.m_value = other.m_data.m_value; } else { - copy_leaf_value(other, *this); + copy_owning(other); } - set_parents(); assert_invariant(); } @@ -2782,6 +2925,22 @@ public: // value access // ////////////////// + /// out-of-line part of the copy constructor for owning types + JSON_HEDLEY_NEVER_INLINE + void copy_owning(const basic_json& other) + { + if (m_data.m_type == value_t::object || m_data.m_type == value_t::array) + { + // copying the container directly would call this constructor again + // for every element, once per nesting level + copy_structured(other); + } + else + { + copy_leaf_value(other, *this); + } + } + /// get a pointer to the value (object) object_t* get_impl_ptr(object_t* /*unused*/) noexcept { @@ -2842,14 +3001,16 @@ public: return m_data.m_type == value_t::number_integer ? &m_data.m_value.number_integer : nullptr; } - /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + /// get a pointer to the value (unsigned number); takes a placeholder if + /// number_unsigned_t is number_integer_t (the integer overload handles it then) + number_unsigned_t* get_impl_ptr(detail::number_unsigned_overload_type* /*unused*/) noexcept { return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } - /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + /// get a pointer to the value (unsigned number); takes a placeholder if + /// number_unsigned_t is number_integer_t (the integer overload handles it then) + constexpr const number_unsigned_t* get_impl_ptr(const detail::number_unsigned_overload_type* /*unused*/) const noexcept { return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } @@ -2878,6 +3039,40 @@ public: return is_binary() ? m_data.m_value.binary : nullptr; } + /// out-of-line throw helpers (message construction kept out of the hot paths) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_iterator_range_of_null(const basic_json* obj) + { + JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", obj->type_name()), obj)); + } + + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_incompatible_reference_type(const basic_json& obj) + { + JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); + } + + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_array_index_exceeds_size_type(size_type idx) + { + JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); + } + + /// @throw out_of_range.401 + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_array_index_out_of_range(size_type idx, const basic_json& j) + { + JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + } + + /// @throw out_of_range.403 + template + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_key_not_found(const KeyType& key, const basic_json& j) + { + JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key, "' not found"), &j)); + } + /*! @brief helper function to implement get_ref() @@ -2900,7 +3095,7 @@ public: return *ptr; } - JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); + throw_incompatible_reference_type(obj); } public: @@ -3348,7 +3543,7 @@ public: // std::map or ordered_map) never moves from its argument, so key is still // valid here regardless of whether KeyType was deduced as an rvalue reference // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved) - JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key_for_message(key), "' not found"), &j)); + throw_key_not_found(key_for_message(key), j); } return it->second; } @@ -3385,7 +3580,7 @@ public: if (JSON_HEDLEY_UNLIKELY(idx >= j.m_data.m_value.array->size())) { - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + throw_array_index_out_of_range(idx, j); } return (*j.m_data.m_value.array)[idx]; @@ -3440,7 +3635,7 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(size_type idx) { - return set_parent(array_at(*this, idx)); + return set_parent_at(array_at(*this, idx)); } /// @brief access specified array element with bounds checking @@ -3454,7 +3649,7 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(const typename object_t::key_type& key) { - return set_parent(object_at(*this, key)); + return set_parent_at(object_at(*this, key)); } /// @brief access specified object element with bounds checking @@ -3463,7 +3658,7 @@ public: detail::is_usable_as_basic_json_key_type::value, int> = 0> reference at(KeyType && key) { - return set_parent(object_at(*this, std::forward(key))); + return set_parent_at(object_at(*this, std::forward(key))); } /// @brief access specified object element with bounds checking @@ -3503,18 +3698,18 @@ public: // resize() rejects other indices that are too large to represent if (JSON_HEDLEY_UNLIKELY(idx == (std::numeric_limits::max)())) { - JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); + throw_array_index_exceeds_size_type(idx); } #if JSON_DIAGNOSTICS // remember array size & capacity before resizing const auto old_size = m_data.m_value.array->size(); - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); #endif m_data.m_value.array->resize(idx + 1); #if JSON_DIAGNOSTICS if (JSON_HEDLEY_UNLIKELY(old_capacity == detail::unknown_size() - || array_capacity() != old_capacity)) + || container_capacity() != old_capacity)) { // capacity has changed: update all parents set_parents(); @@ -3561,8 +3756,9 @@ public: // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { + const auto old_capacity = container_capacity(); auto result = m_data.m_value.object->emplace(std::move(key), nullptr); - return set_parent(result.first->second); + return set_parent_after_insert(result.first->second, old_capacity); } throw_subscript_wrong_type("string"); @@ -3612,8 +3808,9 @@ public: // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { + const auto old_capacity = container_capacity(); auto result = m_data.m_value.object->emplace(lookup_key(std::forward(key)), nullptr); - return set_parent(result.first->second); + return set_parent_after_insert(result.first->second, old_capacity); } throw_subscript_wrong_type("string"); @@ -3839,7 +4036,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(this != pos.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } IteratorType result = end(); @@ -3855,7 +4052,7 @@ public: { if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_out_of_range, "iterator out of range", this)); + detail::throw_literal(detail::exception_id::iterator_out_of_range, "iterator out of range", this); } m_data.m_value.destroy(m_data.m_type); @@ -3897,7 +4094,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_from_other_value, "iterators do not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_range_from_other_value, "iterators do not fit current value", this); } IteratorType result = end(); @@ -3914,7 +4111,7 @@ public: if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() || !last.m_it.primitive_iterator.is_end())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_out_of_range, "iterators out of range", this)); + detail::throw_literal(detail::exception_id::iterator_range_out_of_range, "iterators out of range", this); } m_data.m_value.destroy(m_data.m_type); @@ -4013,7 +4210,7 @@ public: { if (JSON_HEDLEY_UNLIKELY(idx >= size())) { - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); + throw_array_index_out_of_range(idx, *this); } m_data.m_value.array->erase(m_data.m_value.array->begin() + static_cast(idx)); @@ -4519,9 +4716,9 @@ public: } // add the element to the array (move semantics) - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->push_back(std::move(val)); - set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + set_parent_after_insert(m_data.m_value.array->back(), old_capacity); // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor } @@ -4550,9 +4747,9 @@ public: } // add the element to the array - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->push_back(val); - set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + set_parent_after_insert(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an array @@ -4580,8 +4777,9 @@ public: } // add the element to the object + const auto old_capacity = container_capacity(); auto res = m_data.m_value.object->insert(val); - set_parent(res.first->second); + set_parent_after_insert(res.first->second, old_capacity); } /// @brief add an object to an object @@ -4634,9 +4832,9 @@ public: } // add the element to the array (perfect forwarding) - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->emplace_back(std::forward(args)...); - return set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + return set_parent_after_insert(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an object if key does not exist @@ -4657,8 +4855,9 @@ public: } // add the element to the array (perfect forwarding) + const auto old_capacity = container_capacity(); auto res = m_data.m_value.object->emplace(std::forward(args)...); - set_parent(res.first->second); + set_parent_after_insert(res.first->second, old_capacity); // create a result iterator and set iterator to the result of emplace auto it = begin(); @@ -4699,7 +4898,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // insert to array and return iterator @@ -4719,7 +4918,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // moving into a local first keeps this safe even if val aliases @@ -4741,7 +4940,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // insert to array and return iterator @@ -4764,24 +4963,24 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } if (JSON_HEDLEY_UNLIKELY(first.m_object == this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_into_itself, "passed iterators may not belong to container", this)); + detail::throw_literal(detail::exception_id::insert_range_into_itself, "passed iterators may not belong to container", this); } // passed iterators must belong to arrays if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_array())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterators first and last must point to arrays", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to arrays", this); } // insert to array and return iterator @@ -4801,7 +5000,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // copy the values first: ilist may refer to elements of this array @@ -4829,13 +5028,13 @@ public: // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } // passed iterators must belong to objects if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this); } m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); @@ -4870,7 +5069,7 @@ public: // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } // passed iterators must belong to objects @@ -4957,9 +5156,9 @@ public: continue; } } - // set_parent() also repairs the other members, which ordered_json - // relocates when adding a key makes its vector grow - set_parent(m_data.m_value.object->operator[](it.key()) = std::move(it.value())); + // operator[] repairs the other members if the object type moves + // them; assigning through it keeps this O(1) per key + operator[](it.key()) = std::move(it.value()); } } @@ -5009,9 +5208,8 @@ public: continue; } } - // set_parent() also repairs the other members, which ordered_json - // relocates when adding a key makes its vector grow - target->set_parent(target->m_data.m_value.object->operator[](first.key()) = std::move(first.value())); + // operator[] repairs the other members if the object type moves them + target->operator[](first.key()) = std::move(first.value()); ++first; } } @@ -5153,7 +5351,22 @@ public: /// @name lexicographical comparison operators /// @{ - // note parentheses around operands are necessary; see + // the same-type cases for non-owning scalars, shared by JSON_IMPLEMENT_OPERATOR + // and JSON_IMPLEMENT_SCALAR_FAST +#define JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + case value_t::null: \ + return (null_result); \ + case value_t::boolean: \ + return lhs.m_data.m_value.boolean op rhs.m_data.m_value.boolean; \ + case value_t::number_integer: \ + return lhs.m_data.m_value.number_integer op rhs.m_data.m_value.number_integer; \ + case value_t::number_unsigned: \ + return lhs.m_data.m_value.number_unsigned op rhs.m_data.m_value.number_unsigned; \ + case value_t::number_float: \ + return lhs.m_data.m_value.number_float op rhs.m_data.m_value.number_float; + + // note parentheses around the container operands are necessary, because + // GCC otherwise parses `array <` as std::array; see // https://github.com/nlohmann/json/issues/1530 // Mixed signed/unsigned integer comparisons check whether the signed value // is negative before casting. If it is, the comparison is performed with @@ -5189,24 +5402,11 @@ public: return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ } \ \ - case value_t::null: \ - return (null_result); \ - \ + JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + \ case value_t::string: \ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ - case value_t::boolean: \ - return (lhs.m_data.m_value.boolean) op (rhs.m_data.m_value.boolean); \ - \ - case value_t::number_integer: \ - return (lhs.m_data.m_value.number_integer) op (rhs.m_data.m_value.number_integer); \ - \ - case value_t::number_unsigned: \ - return (lhs.m_data.m_value.number_unsigned) op (rhs.m_data.m_value.number_unsigned); \ - \ - case value_t::number_float: \ - return (lhs.m_data.m_value.number_float) op (rhs.m_data.m_value.number_float); \ - \ case value_t::binary: \ return (*lhs.m_data.m_value.binary) op (*rhs.m_data.m_value.binary); \ \ @@ -5250,6 +5450,24 @@ public: \ return (default_result); + // inline fast path for two values of the same non-owning scalar type; anything + // else falls through to the out-of-line general comparison +#define JSON_IMPLEMENT_SCALAR_FAST(op, null_result) \ + if (lhs.m_data.m_type == rhs.m_data.m_type) \ + { \ + switch (lhs.m_data.m_type) \ + { \ + JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + case value_t::object: \ + case value_t::array: \ + case value_t::string: \ + case value_t::binary: \ + case value_t::discarded: \ + default: \ + break; \ + } \ + } + JSON_PRIVATE_UNLESS_TESTED: // returns true if: // - any operand is NaN and the other operand is of number type @@ -5277,6 +5495,173 @@ public: return compares_unordered(*this, rhs, inverse); } + /// @brief whether objects and arrays are the standard containers, whose + /// elements equal_bounded() may therefore compare itself + using standard_object_t = std::map>>; + using standard_array_t = std::vector>; + using equal_bounded_containers = detail::conjunction, std::is_same>; + + // out-of-line general comparisons; the operators below handle same-type + // scalars inline and fall through to these + JSON_HEDLEY_NEVER_INLINE + static bool equal_general(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + JSON_IMPLEMENT_OPERATOR( ==, true, false, false, + compare_iteratively(lhs, rhs, false) == compare_result::equal, true) +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + } + + JSON_HEDLEY_NEVER_INLINE + static bool less_general(const_reference lhs, const_reference rhs) noexcept + { + // default_result is used if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + compare_iteratively(lhs, rhs, true) == compare_result::less, false) + } + +#if JSON_HAS_THREE_WAY_COMPARISON + JSON_HEDLEY_NEVER_INLINE + static std::partial_ordering three_way_general(const_reference lhs, const_reference rhs) noexcept // *NOPAD* + { + // default_result is used if we cannot compare values. In that case, + // we compare types. + JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* + std::partial_ordering::equivalent, + std::partial_ordering::unordered, + lhs_type <=> rhs_type, // *NOPAD* + to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) + } +#endif + + /// @brief compare two leaves of the same type inside equal_structured() + /// @return 1 if they are equal, 0 if they are not, and -1 if the values are + /// of different types or are not handled here, which this leaves to + /// equal_structured() + /// @note Unlike JSON_IMPLEMENT_SCALAR_FAST, this also compares strings: + /// they are the most common leaves in a document, so comparing them + /// inline pays off in the container walk, while the operators keep + /// their inlined code at each call site small. Binary values are + /// rare and left to equal_general(). + JSON_HEDLEY_ALWAYS_INLINE + static int equal_leaf(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + if (lhs.m_data.m_type == rhs.m_data.m_type) + { + switch (lhs.m_data.m_type) + { + case value_t::null: + return 1; + case value_t::boolean: + return lhs.m_data.m_value.boolean == rhs.m_data.m_value.boolean; + case value_t::number_integer: + return lhs.m_data.m_value.number_integer == rhs.m_data.m_value.number_integer; + case value_t::number_unsigned: + return lhs.m_data.m_value.number_unsigned == rhs.m_data.m_value.number_unsigned; + case value_t::number_float: + return lhs.m_data.m_value.number_float == rhs.m_data.m_value.number_float; + case value_t::string: + return *lhs.m_data.m_value.string == *rhs.m_data.m_value.string; + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + } +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + return -1; + } + + /// @brief compare for equality, walking objects and arrays here instead of + /// in their containers so that the depth is a parameter and needs no + /// thread-local count; only used for the standard containers + static bool equal_bounded(const_reference lhs, const_reference rhs, std::size_t depth) noexcept + { + const int leaf = equal_leaf(lhs, rhs); + return leaf < 0 ? equal_structured(lhs, rhs, depth) : leaf != 0; + } + + static bool equal_structured(const_reference lhs, const_reference rhs, std::size_t depth) noexcept + { + const auto type = lhs.m_data.m_type; + if (type != rhs.m_data.m_type || (type != value_t::array && type != value_t::object)) + { + return equal_general(lhs, rhs); + } + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + return compare_iteratively(lhs, rhs, false) == compare_result::equal; + } + + if (type == value_t::array) + { + const auto& a = *lhs.m_data.m_value.array; + const auto& b = *rhs.m_data.m_value.array; + if (a.size() != b.size()) + { + return false; + } + auto it_b = b.cbegin(); + for (auto it_a = a.cbegin(); it_a != a.cend(); ++it_a, ++it_b) + { + const int leaf = equal_leaf(*it_a, *it_b); + if (leaf < 0 ? !equal_structured(*it_a, *it_b, depth + 1) : leaf == 0) + { + return false; + } + } + return true; + } + + const auto& a = *lhs.m_data.m_value.object; + const auto& b = *rhs.m_data.m_value.object; + if (a.size() != b.size()) + { + return false; + } + auto it_b = b.cbegin(); + for (auto it_a = a.cbegin(); it_a != a.cend(); ++it_a, ++it_b) + { + if (!(it_a->first == it_b->first)) + { + return false; + } + const int leaf = equal_leaf(it_a->second, it_b->second); + if (leaf < 0 ? !equal_structured(it_a->second, it_b->second, depth + 1) : leaf == 0) + { + return false; + } + } + return true; + } + + static bool equal_dispatch(const_reference lhs, const_reference rhs, std::true_type /*standard containers*/) noexcept + { + return equal_bounded(lhs, rhs, 0); + } + + static bool equal_dispatch(const_reference lhs, const_reference rhs, std::false_type /*other containers*/) noexcept + { + return equal_general(lhs, rhs); + } + public: #if JSON_HAS_THREE_WAY_COMPARISON /// @brief comparison: equal @@ -5288,11 +5673,11 @@ public: JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif const_reference lhs = *this; - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, - compare_iteratively(lhs, rhs, false) == compare_result::equal, true) + JSON_IMPLEMENT_SCALAR_FAST( ==, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP #endif + return equal_dispatch(lhs, rhs, equal_bounded_containers {}); } /// @brief comparison: equal @@ -5309,13 +5694,8 @@ public: std::partial_ordering operator<=>(const_reference rhs) const noexcept // *NOPAD* { const_reference lhs = *this; - // default_result is used if we cannot compare values. In that case, - // we compare types. - JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* - std::partial_ordering::equivalent, - std::partial_ordering::unordered, - lhs_type <=> rhs_type, // *NOPAD* - to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) + JSON_IMPLEMENT_SCALAR_FAST(<=>, std::partial_ordering::equivalent) // *NOPAD* + return three_way_general(lhs, rhs); } /// @brief comparison: 3-way @@ -5403,11 +5783,11 @@ public: JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, - compare_iteratively(lhs, rhs, false) == compare_result::equal, true) + JSON_IMPLEMENT_SCALAR_FAST( ==, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP #endif + return equal_dispatch(lhs, rhs, equal_bounded_containers {}); } // The friend comparisons with a scalar name the JSON type via decltype of @@ -5462,11 +5842,8 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ friend bool operator<(const_reference lhs, const_reference rhs) noexcept { - // default_result is used if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), - compare_iteratively(lhs, rhs, true) == compare_result::less, false) + JSON_IMPLEMENT_SCALAR_FAST( <, false) + return less_general(lhs, rhs); } /// @brief comparison: less than @@ -5577,6 +5954,8 @@ public: #endif #undef JSON_IMPLEMENT_OPERATOR +#undef JSON_IMPLEMENT_SCALAR_FAST +#undef JSON_IMPLEMENT_SCALAR_CASES /// @} @@ -6557,24 +6936,18 @@ public: }; // wrapper for "add" operation; add value at ptr - const auto operation_add = [&result](json_pointer & ptr, const basic_json & val) + const auto operation_add = [&result](json_pointer & ptr, basic_json val) { // adding to the root of the target document means replacing it if (ptr.empty()) { - result = val; + result = std::move(val); return; } - // make sure the top element of the pointer exists - json_pointer const top_pointer = ptr.top(); - if (top_pointer != ptr) - { - result.at(top_pointer); - } - - // get reference to the parent of the JSON pointer ptr - const auto last_path = ptr.back(); + // get reference to the parent of the JSON pointer ptr; a missing + // parent (or one of its ancestors) is reported by at() + auto last_path = std::move(ptr.reference_tokens.back()); ptr.pop_back(); // parent must exist when performing patch add per RFC6902 specs basic_json& parent = result.at(ptr); @@ -6585,7 +6958,7 @@ public: case value_t::object: { // use operator[] to add value - parent[last_path] = val; + parent[std::move(last_path)] = std::move(val); break; } @@ -6594,7 +6967,7 @@ public: if (last_path == "-") { // special case: append to back - parent.push_back(val); + parent.push_back(std::move(val)); } else { @@ -6602,11 +6975,11 @@ public: if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) { // avoid undefined behavior - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &parent)); + throw_array_index_out_of_range(idx, parent); } // default case: insert add offset - parent.insert(parent.begin() + static_cast(idx), val); + parent.insert(parent.begin() + static_cast(idx), std::move(val)); } break; } @@ -6621,7 +6994,7 @@ public: case value_t::binary: case value_t::discarded: default: - JSON_THROW(out_of_range::create(detail::exception_id::patch_add_parent_not_container, detail::concat("cannot add value: the JSON Patch 'add' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + detail::throw_patch_add_parent(parent); } }; @@ -6629,7 +7002,8 @@ public: const auto operation_remove = [this, & result](json_pointer & ptr) { // get reference to the parent of the JSON pointer ptr - const auto last_path = ptr.back(); + static_cast(ptr.back()); // throws out_of_range.405 if ptr is the root + const auto last_path = std::move(ptr.reference_tokens.back()); ptr.pop_back(); basic_json& parent = result.at(ptr); @@ -6644,7 +7018,7 @@ public: } else { - JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", last_path, "' not found"), this)); + throw_key_not_found(last_path, *this); } } else if (parent.is_array()) @@ -6656,7 +7030,7 @@ public: { // the parent of a "remove" target must be an object or array // (see #5396) - JSON_THROW(out_of_range::create(detail::exception_id::patch_remove_parent_not_container, detail::concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + detail::throw_patch_remove_parent(parent); } }; @@ -6687,7 +7061,7 @@ public: // type check: top level value must be an array if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array())) { - JSON_THROW(parse_error::create(detail::exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &json_patch)); + detail::throw_patch_not_an_array(json_patch); } // iterate and apply the operations @@ -6701,21 +7075,16 @@ public: // find value auto it = val.m_data.m_value.object->find(member); - // context-sensitive error message - const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\''); // NOLINT(bugprone-unused-local-non-trivial-variable) - // check if the desired value is present if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end())) { - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat(error_msg, " must have member '", member, "'"), &val)); + detail::throw_patch_missing_member(op, member, false, val); } // check if the result is of type string if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) { - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat(error_msg, " must have string member '", member, "'"), &val)); + detail::throw_patch_missing_member(op, member, true, val); } // no error: return value @@ -6725,12 +7094,12 @@ public: // type check: every element of the array must be an object if (JSON_HEDLEY_UNLIKELY(!val.is_object())) { - JSON_THROW(parse_error::create(detail::exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &val)); + detail::throw_patch_not_an_array(val); } // collect mandatory members - const auto op = get_value("op", "op", true).template get(); - const auto path = get_value(op, "path", true).template get(); + const string_t& op = *get_value("op", "op", true).m_data.m_value.string; + const string_t& path = *get_value(op, "path", true).m_data.m_value.string; json_pointer ptr(path); switch (get_op(op)) @@ -6756,38 +7125,37 @@ public: case patch_operations::move: { - const auto from_path = get_value("move", "from", true).template get(); + const string_t& from_path = *get_value("move", "from", true).m_data.m_value.string; json_pointer from_ptr(from_path); if (JSON_HEDLEY_UNLIKELY(is_proper_prefix(from_ptr, ptr))) { - JSON_THROW(out_of_range::create(detail::exception_id::patch_move_into_child, detail::concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); + detail::throw_patch_move_into_child(from_path, path, result); } // the "from" location must exist - use at() - basic_json const v = result.at(from_ptr); + // a root "from" is rejected by operation_remove below, which must leave result intact + basic_json v = from_ptr.empty() ? basic_json(result) : std::move(result.at(from_ptr)); // The move operation is functionally identical to a // "remove" operation on the "from" location, followed // immediately by an "add" operation at the target // location with the value that was just removed. operation_remove(from_ptr); - operation_add(ptr, v); + operation_add(ptr, std::move(v)); break; } case patch_operations::copy: { - const auto from_path = get_value("copy", "from", true).template get(); + const string_t& from_path = *get_value("copy", "from", true).m_data.m_value.string; const json_pointer from_ptr(from_path); - // the "from" location must exist - use at() - basic_json const v = result.at(from_ptr); - // The copy is functionally identical to an "add" // operation at the target location using the value - // specified in the "from" member. - operation_add(ptr, v); + // specified in the "from" member; the "from" location + // must exist - use at() + operation_add(ptr, result.at(from_ptr)); break; } @@ -6808,7 +7176,7 @@ public: // throw an exception if the test fails if (JSON_HEDLEY_UNLIKELY(!success)) { - JSON_THROW(other_error::create(detail::exception_id::patch_test_failed, detail::concat("unsuccessful: ", val.dump()), &val)); + detail::throw_patch_test_failed(val); } break; @@ -6819,7 +7187,7 @@ public: { // op must be "add", "remove", "replace", "move", "copy", or // "test" - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat("operation value '", op, "' is invalid"), &val)); + detail::throw_patch_invalid_operation(op, val); } } } @@ -6873,12 +7241,11 @@ public: std::size_t index = 0; /// objects: the next member of source to look at const_iterator member{}; // NOLINT(readability-redundant-member-init) - /// objects: the keys common to both, in source's order - std::vector common_keys{}; // NOLINT(readability-redundant-member-init) - /// objects: the next entry of common_keys - std::size_t next_common = 0; - /// objects: the "add" operations for keys only target has - basic_json added_ops{}; // NOLINT(readability-redundant-member-init) + /// objects: for each member of source (in order), its value in + /// target, or nullptr if target has no such key; `index` is the next + std::vector target_values{}; // NOLINT(readability-redundant-member-init) + /// objects: the members only target has, in target's order + std::vector added{}; // NOLINT(readability-redundant-member-init) }; // The operations of a diff are built by the functions below rather than @@ -6934,111 +7301,113 @@ public: } } + /// @brief appends the "add" operations for the members @a added to @a result + static void diff_added(basic_json& result, const std::vector& added, const string_t& path) + { + for (const auto& it : added) + { + diff_add(result, detail::concat(path, '/', detail::escape(it.key())), it.value()); + } + } + /*! @brief compare the keys of objects @a source and @a target If object_t does not keep its members in insertion order, or if the keys both objects have are in the same order in both, and the keys only - @a target has come after them, stores the keys common to both in - source's order in @a common_keys, stores the "add" operations for the keys - only @a target has in @a added_ops, and returns true: the caller then diffs - the objects member by member. Otherwise, appends operations that remove - every member of @a source and add every member of @a target to @a result, - and returns false. + @a target has come after them, stores in @a target_values, for each member + of source in order, its value in target (nullptr for a key target does not + have), stores the members only target has in @a added, and returns true: + the caller then diffs the objects member by member. Otherwise, appends + operations that remove every member of @a source and add every member of + @a target to @a result, and returns false. */ static bool diff_object_keys(basic_json& result, const basic_json& source, const basic_json& target, - const string_t& path, std::vector& common_keys, - basic_json& added_ops) + const string_t& path, std::vector& target_values, + std::vector& added) { - // first pass: record, for every source key, whether it is - // common to both objects (in source's iteration order) or - // was deleted (i.e., in source but not in target) -- this is - // a by-product of the target.find() call already needed to - // tell the two cases apart, so it adds no extra lookups. The - // "remove" ops themselves are emitted later, interleaved - // with the per-key diffs in the caller's fast path, to match - // source's original iteration order (as the original, - // pre-reordering-aware implementation did) instead of - // grouping all removes before all per-key diffs. - std::vector common_keys_source_order; + // first pass: one lookup per source key + target_values.reserve(source.size()); + std::size_t common = 0; for (auto it = source.cbegin(); it != source.cend(); ++it) { - if (target.find(it.key()) != target.end()) + const auto found = target.find(it.key()); + if (found != target.cend()) { - common_keys_source_order.push_back(it.key()); - } - } - - // second pass: find keys that were added (i.e., in target but - // not in source), and record the keys common to both, in - // target's iteration order -- again a by-product of the - // source.find() call already needed to detect added keys. At - // the same time, determine whether every added key comes - // after every common key in target's order (a precondition - // for the fast path, which only ever appends new keys - // at the very end). Both are only needed for an object_t that - // keeps its members in insertion order, such as the one - // backing `ordered_json`; for any other object_t, the fast - // path is always taken and they are not computed. - // The patch ops for keys that were added (i.e., in target but not - // in source) are built here so the fast path can reuse - // them without a second source.find() per target key. Only - // used by the fast path -- the slow (reordering) path - // rebuilds "add" ops for every key itself. - std::vector common_keys_target_order; - bool new_keys_form_suffix = true; - bool seen_new_key = false; - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - if (source.find(it.key()) == source.end()) - { - seen_new_key = true; - diff_add(added_ops, detail::concat(path, '/', detail::escape(it.key())), it.value()); + target_values.push_back(&found.value()); + ++common; } else { -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - common_keys_target_order.push_back(it.key()); - if (seen_new_key) - { - new_keys_form_suffix = false; - } - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif + target_values.push_back(nullptr); } } - // Only an object type that keeps its members in insertion - // order, such as nlohmann::ordered_map, can need reordering: - // patch() appends a new member at the end of such an object. - // Any other object type places its members itself - std::map - // in key order, a hash map in an order its operator== ignores - - // so a member-by-member diff always reproduces target there. + // second pass: find the keys only target has. An object type that + // keeps its members in insertion order, such as the one backing + // ordered_json, also needs the common keys to be in the same order + // in both objects, and the new keys to come after them, as patch() + // appends a new member at the end of such an object. Any other + // object type places its members itself, so a member-by-member diff + // always reproduces target there. #ifdef JSON_HEDLEY_MSVC_VERSION #pragma warning(push ) #pragma warning(disable : 4127) // ignore warning to replace if with if constexpr #endif - if (!detail::is_ordered_map::value - || (common_keys_source_order == common_keys_target_order && new_keys_form_suffix)) + bool fast_path = true; + if (detail::is_ordered_map::value) { - // fast path: order of common keys already matches (or the - // object_t's iteration order does not depend on - // insertion history), so a plain per-key diff is correct - // and minimal, as before - common_keys = std::move(common_keys_source_order); - return true; + // walk the common keys of source in lockstep with target: a key + // of target that is not the next one is either new or out of order + auto src = source.cbegin(); + std::size_t next = 0; + bool seen_new = false; + for (auto it = target.cbegin(); it != target.cend() && fast_path; ++it) + { + while (next < target_values.size() && target_values[next] == nullptr) + { + ++src; + ++next; + } + + if (next < target_values.size() && src.key() == it.key()) + { + // a common key after a new one: new keys are no suffix + fast_path = !seen_new; + ++src; + ++next; + } + else if (source.find(it.key()) == source.cend()) + { + seen_new = true; + added.push_back(it); + } + else + { + // a common key out of order + fast_path = false; + } + } + } + else if (target.size() != common) + { + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.cend()) + { + added.push_back(it); + } + } } #ifdef JSON_HEDLEY_MSVC_VERSION #pragma warning( pop ) #endif + if (fast_path) + { + return true; + } + // slow path: the common keys are in a different relative // order in source and target (only possible for a // reorderable object_t like ordered_map). Building a @@ -7088,6 +7457,14 @@ public: return; } + diff_unequal(result, source, target, path, depth); + } + + /// @brief @ref diff_recursively for values known to be unequal; callers + /// compare first so they build no path for equal ones + static void diff_unequal(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, const std::size_t depth) + { if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) { diff_iteratively(result, source, target, path); @@ -7110,7 +7487,10 @@ public: while (i < source.size() && i < target.size()) { // recursive call to compare array values at index i - diff_recursively(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); + if (!(source[i] == target[i])) + { + diff_unequal(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); + } ++i; } @@ -7122,25 +7502,24 @@ public: case value_t::object: { - std::vector common_keys; - basic_json added_ops(value_t::array); - if (diff_object_keys(result, source, target, path, common_keys, added_ops)) + std::vector target_values; + std::vector added; + if (diff_object_keys(result, source, target, path, target_values, added)) { - // fast path: common_keys is, by construction, the - // subsequence of source's keys that are common to both - // objects, in source's iteration order -- so it can be - // walked in lockstep with `source` using a cheap key - // comparison instead of another lookup. Deleted keys - // (those source keys not in common_keys) are interleaved - // here too, in source's original order, to match the + // fast path: target_values holds, for every member of + // source in order, its value in target -- so no further + // lookup is needed. Deleted keys are interleaved here + // too, in source's original order, to match the // historical (pre-reordering-aware) output order. - auto common_it = common_keys.cbegin(); - for (auto it = source.cbegin(); it != source.cend(); ++it) + std::size_t i = 0; + for (auto it = source.cbegin(); it != source.cend(); ++it, ++i) { - if (common_it != common_keys.cend() && it.key() == *common_it) + if (target_values[i] != nullptr) { - diff_recursively(result, it.value(), target[it.key()], detail::concat(path, '/', detail::escape(it.key())), depth + 1); - ++common_it; + if (!(it.value() == *target_values[i])) + { + diff_unequal(result, it.value(), *target_values[i], detail::concat(path, '/', detail::escape(it.key())), depth + 1); + } } else { @@ -7149,10 +7528,8 @@ public: } } - // append the "add" ops for brand-new keys collected by - // diff_object_keys -- no second source.find() per target - // key needed - result.insert(result.end(), added_ops.begin(), added_ops.end()); + // append the "add" operations for brand-new keys + diff_added(result, added, path); } break; } @@ -7225,17 +7602,17 @@ public: case value_t::object: { - std::vector common_keys; - basic_json added_ops(value_t::array); - if (diff_object_keys(result, s, t, current_path, common_keys, added_ops)) + std::vector target_values; + std::vector added; + if (diff_object_keys(result, s, t, current_path, target_values, added)) { // fast path: the frame walks source in lockstep with // common_keys, as diff_recursively does, and appends // added_ops once all members are done stack.emplace_back(&s, &t, current_path.size()); stack.back().member = s.cbegin(); - stack.back().common_keys = std::move(common_keys); - stack.back().added_ops = std::move(added_ops); + stack.back().target_values = std::move(target_values); + stack.back().added = std::move(added); } return; } @@ -7297,11 +7674,10 @@ public: if (it != s->cend()) { ++stack.back().member; - const std::size_t next_common = stack.back().next_common; - if (next_common < stack.back().common_keys.size() && it.key() == stack.back().common_keys[next_common]) + const basic_json* const target_value_ptr = stack.back().target_values[stack.back().index++]; + if (target_value_ptr != nullptr) { - ++stack.back().next_common; - const basic_json& target_value = (*t)[it.key()]; + const basic_json& target_value = *target_value_ptr; detail::concat_into(current_path, '/', detail::escape(it.key())); enter(it.value(), target_value); if (stack.size() == depth) @@ -7319,7 +7695,7 @@ public: // append the "add" ops for brand-new keys collected when the // object was entered - result.insert(result.end(), stack.back().added_ops.begin(), stack.back().added_ops.end()); + diff_added(result, stack.back().added, current_path); } // this array or object is done: continue with the one it is in @@ -7449,6 +7825,176 @@ public: } public: + /// @brief creates a diff as a JSON Merge Patch + /// @sa https://json.nlohmann.me/api/basic_json/merge_diff/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json merge_diff(const basic_json& source, const basic_json& target) + { + // unless both are objects, the patch is the target itself + if (!source.is_object() || !target.is_object()) + { + return target; + } + + return merge_diff_recursively(source, target, 0); + } + + private: + /// @brief two objects @ref merge_diff_iteratively is diffing + struct merge_diff_frame + { + merge_diff_frame(const basic_json* source_, const basic_json* target_) + : source(source_), target(target_), member(source_->cbegin()), patch(value_t::object) + {} + + // declared for GCC's -Weffc++, as in diff_frame + merge_diff_frame(const merge_diff_frame&) = default; + merge_diff_frame(merge_diff_frame&&) = default; + merge_diff_frame& operator=(const merge_diff_frame&) = default; + merge_diff_frame& operator=(merge_diff_frame&&) = default; + ~merge_diff_frame() = default; + + /// the objects being diffed + const basic_json* source; + const basic_json* target; + /// the member of source being diffed; advanced once it is done + const_iterator member; + /// the patch for the members of source diffed so far + basic_json patch; + }; + + /// @brief add the members of object @a target that @a source does not + /// have to the merge patch @a result + static void merge_diff_added_members(basic_json& result, const basic_json& source, const basic_json& target) + { + for (auto it = target.begin(); it != target.end(); ++it) + { + auto itf = source.find(it.key()); + if (itf == source.end()) + { + result[it.key()] = it.value(); + } + } + } + + /*! + @brief @ref merge_diff of two objects at nesting level @a depth + + Diffing two nested objects calls this function again, once per nesting + level, so values nested deeply enough would exhaust the call stack and + terminate the process. The descent is bounded here: once @ref + detail::recursion_depth_limit levels have been entered, @ref + merge_diff_iteratively diffs what is left without the call stack. + */ + static basic_json merge_diff_recursively(const basic_json& source, const basic_json& target, const std::size_t depth) + { + JSON_ASSERT(source.is_object() && target.is_object()); + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + return merge_diff_iteratively(source, target); + } + + basic_json result(value_t::object); + + for (auto it = source.begin(); it != source.end(); ++it) + { + auto itf = target.find(it.key()); + if (itf != target.end()) + { + if (it.value().is_object() && itf.value().is_object()) + { + auto diff = merge_diff_recursively(it.value(), itf.value(), depth + 1); + if (!diff.empty()) + { + result[it.key()] = std::move(diff); + } + } + else if (it.value() != itf.value()) + { + result[it.key()] = itf.value(); + } + } + else + { + result[it.key()] = value_t::null; + } + } + + merge_diff_added_members(result, source, target); + return result; + } + + /*! + @brief @ref merge_diff of two objects without the call stack + + Produces the same patch as @ref merge_diff_recursively, keeping the objects + being diffed on an explicit stack instead of descending into them. Two + nested objects are still diffed completely before the next member, so the + patch has its members in the same order as in the recursive version. Only + reached for objects nested more deeply than @ref + detail::recursion_depth_limit. + */ + static basic_json merge_diff_iteratively(const basic_json& source, const basic_json& target) + { + // the patch of source and target, set once their frame is popped + basic_json result; + std::vector stack; + stack.emplace_back(&source, &target); + while (!stack.empty()) + { + // the frame's members are copied out and it is changed through + // stack.back(): pushing a frame for nested objects or popping + // this one would invalidate a reference to it + const basic_json* const s = stack.back().source; + const basic_json* const t = stack.back().target; + const const_iterator it = stack.back().member; + + if (it != s->cend()) + { + const const_iterator itf = t->find(it.key()); + if (itf != t->cend()) + { + if (it.value().is_object() && itf.value().is_object()) + { + // member is advanced once their patch is under its key + stack.emplace_back(&it.value(), &itf.value()); + continue; + } + if (it.value() != itf.value()) + { + stack.back().patch[it.key()] = itf.value(); + } + } + else + { + stack.back().patch[it.key()] = value_t::null; + } + ++stack.back().member; + continue; + } + + merge_diff_added_members(stack.back().patch, *s, *t); + + // these objects are done: continue with the ones they are in + basic_json patch = std::move(stack.back().patch); + stack.pop_back(); + if (stack.empty()) + { + result = std::move(patch); + } + else + { + if (!patch.empty()) + { + stack.back().patch[stack.back().member.key()] = std::move(patch); + } + ++stack.back().member; + } + } + return result; + } + /// @} }; @@ -7625,7 +8171,7 @@ struct formatter // NOLINT(cert-dcl58-c // where needed. #ifndef JSON_NO_AUTOMATIC_UDLS // NOLINTNEXTLINE(misc-header-include-cycle): json_literals.hpp includes this header - #include + #include // IWYU pragma: export #endif #endif // INCLUDE_NLOHMANN_JSON_HPP_ diff --git a/include/nlohmann/ordered_map.hpp b/include/nlohmann/ordered_map.hpp index fdcf4a5c0..10fc764ea 100644 --- a/include/nlohmann/ordered_map.hpp +++ b/include/nlohmann/ordered_map.hpp @@ -90,6 +90,15 @@ private: return self.end(); } + /// @brief throw the exception of a failed at() lookup + /// @note kept out of line so that at_impl() stays small enough to inline + /// @throw std::out_of_range always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_key_not_found() + { + JSON_THROW(std::out_of_range("key not found")); + } + /// @brief shared implementation of the const and non-const at() overloads /// @throw std::out_of_range if @a key is not found template @@ -98,7 +107,7 @@ private: const auto it = find_impl(self, key); if (it == self.end()) { - JSON_THROW(std::out_of_range("key not found")); + throw_key_not_found(); } return it->second; } @@ -143,6 +152,17 @@ public: return {std::prev(this->end()), true}; } + /// @brief add an element at the end without checking whether the key exists + /// @note The caller must guarantee that @a key is not yet contained, otherwise + /// the container holds duplicate keys. This avoids the linear search of + /// emplace() and operator[] when many keys known to be new are added. + template::value, int> = 0> + void emplace_new_key(const key_type& key, V && t) + { + append(key, std::forward(t)); + } + T& operator[](const key_type& key) { return emplace(key, T{}).first->second; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1feaedd97..a44e78616 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -42,6 +42,11 @@ // each of these, not only the forward declarations from json_fwd.hpp, so IWYU's suggestion to // drop them (nothing in this file otherwise names the type) would break every downstream // translation unit that relies on basic_json<>'s defaults actually being usable. +// export: client code names what these headers declare (nlohmann::json and ordered_json from +// json_fwd.hpp, adl_serializer for specializations, json_ref behind brace initialization, the +// NLOHMANN_DEFINE_* and NLOHMANN_JSON_NAMESPACE_* macros, and the literals at the end of this file), and include analyzers must +// attribute it to this header, not tell clients to include the internal ones (issue #5812). +// tests/iwyu_consumer checks this. // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -3344,6 +3349,34 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept } #endif +/*! +@brief whether a value of the given type owns heap memory + +Strings, binaries, objects, and arrays are stored as pointers to allocated +values; all other types are stored directly in the union. Every enumerator is +listed so that -Wswitch-enum flags this function when a new type is added. +*/ +inline bool owns_memory(const value_t t) noexcept +{ + switch (t) + { + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::binary: + return true; + + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + return false; + } +} + /*! @brief compare an integer with a floating point number without precision loss @@ -3421,6 +3454,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // size_t +#include // move // #include @@ -3437,39 +3471,46 @@ namespace detail * Note the order of escaping "~" to "~0" and "/" to "~1" is important. * * The string is rebuilt in a single pass, appending whole runs between the - * characters that need escaping. Scanning with find_first_of() keeps the - * common case -- nothing to escape -- as fast as a single search, while - * repeated replace() calls would move the tail of the string once per - * escaped character. + * characters that need escaping, while repeated replace() calls would move + * the tail of the string once per escaped character. @ref escape_into appends + * the escaped string to an existing one. */ +template +inline void escape_into(StringType& out, const StringType& s) +{ + const std::size_t size = s.size(); + std::size_t run = 0; + for (std::size_t i = 0; i < size; ++i) + { + const char c = s[i]; + if (c == '~' || c == '/') + { + out.append(s.data() + run, i - run); + out.append(c == '~' ? "~0" : "~1", 2); + run = i + 1; + } + } + out.append(s.data() + run, size - run); +} + template inline StringType escape(const StringType& s) { - auto next_special = [&s](std::size_t from) + // a single scan for both special characters; nothing to escape is the common case + const std::size_t size = s.size(); + std::size_t i = 0; + while (i < size && s[i] != '~' && s[i] != '/') { - const auto tilde = s.find_first_of('~', from); - const auto slash = s.find_first_of('/', from); - return tilde < slash ? tilde : slash; // npos is the largest value - }; - - auto pos = next_special(0); - if (pos == StringType::npos) + ++i; + } + if (i == size) { return s; } StringType result; - result.reserve(s.size() + 2); - - std::size_t run = 0; - while (pos != StringType::npos) - { - result.append(s.data() + run, pos - run); - result.append(s[pos] == '~' ? "~0" : "~1", 2); - run = pos + 1; - pos = next_special(run); - } - result.append(s.data() + run, s.size() - run); + result.reserve(size + 2); + escape_into(result, s); return result; } @@ -3514,7 +3555,7 @@ inline void unescape(StringType& s) pos = s.find_first_of('~', run); } result.append(s.data() + run, s.size() - run); - s = result; + s = std::move(result); } } // namespace detail @@ -3741,9 +3782,11 @@ NLOHMANN_JSON_NAMESPACE_END #include // numeric_limits +#include // map, multimap #include // char_traits #include // tuple #include // false_type, is_constructible, is_integral, is_same, true_type +#include // unordered_map, unordered_multimap #include // declval #include // vector #if defined(__cpp_lib_byte) && __cpp_lib_byte >= 201603L @@ -4693,6 +4736,22 @@ struct is_compatible_integer_type : is_compatible_integer_type_impl {}; +// whether to_json() stores an integer of type CompatibleNumberIntegerType as +// number_integer +template +struct is_compatible_number_integer_type + : is_compatible_integer_type {}; + +// if number_unsigned_t is the same type as number_integer_t, unsigned integers +// are stored as number_integer as well (#2573) +template +struct is_compatible_number_integer_type, + std::is_integral, + std::is_unsigned, + negation>>::value>> + : std::is_constructible {}; + template struct is_compatible_type_impl: std::false_type {}; @@ -4907,6 +4966,31 @@ struct has_capacity : std::integral_constant struct is_ordered_map : has_capacity {}; +// whether T is a specialization of nlohmann::ordered_map, which only ever +// appends members, so they can only move when its capacity changes +template +struct is_nlohmann_ordered_map : std::false_type {}; + +template +struct is_nlohmann_ordered_map> : std::true_type {}; + +// whether adding or erasing elements never moves the other elements of the +// container T; only node-based standard containers are known to guarantee this +template +struct has_stable_elements : std::false_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + // to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > T conditional_static_cast(U value) @@ -5030,6 +5114,25 @@ constexpr bool value_in_range_of(T val) template using bool_constant = std::integral_constant; +// Stands in for number_unsigned_t in overload sets if it is the same type as +// number_integer_t, so that the unsigned overloads do not collide with the +// integer ones. Never constructed. Such a basic_json stores every integer as +// number_integer (#2573). +struct same_as_number_integer_type {}; + +// NumberUnsignedType, or same_as_number_integer_type if it is the same type as +// NumberIntegerType; basic_json uses this one, as it cannot name its own member +// types through an incomplete basic_json (MSVC reports a recursive alias, and +// old GCC versions crash) +template +using number_unsigned_overload_type = typename std::conditional::value, + same_as_number_integer_type, NumberUnsignedType>::type; + +// number_unsigned_t of BasicJsonType, or same_as_number_integer_type if it is +// the same type as number_integer_t +template +using number_unsigned_overload_t = number_unsigned_overload_type; + /////////////////////////////////////////////////////////////////////////////// // is_c_string /////////////////////////////////////////////////////////////////////////////// @@ -5635,7 +5738,7 @@ void templated_json_throw(ExceptionType exception) @throw type_error.302 always */ template -JSON_HEDLEY_NO_RETURN inline void throw_type_must_be(const char* expected, const BasicJsonType& j) +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_type_must_be(const char* expected, const BasicJsonType& j) { static_cast(expected); // unused when JSON_NOEXCEPTION is defined static_cast(j); @@ -5650,7 +5753,7 @@ JSON_HEDLEY_NO_RETURN inline void throw_type_must_be(const char* expected, const @throw type_error always */ template -JSON_HEDLEY_NO_RETURN inline void throw_cannot_use_with(const exception_id id_, const char* operation, const BasicJsonType& j) +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_cannot_use_with(const exception_id id_, const char* operation, const BasicJsonType& j) { static_cast(id_); // unused when JSON_NOEXCEPTION is defined static_cast(operation); @@ -5658,6 +5761,77 @@ JSON_HEDLEY_NO_RETURN inline void throw_cannot_use_with(const exception_id id_, JSON_THROW(type_error::create(id_, concat("cannot use ", operation, " with ", j.type_name()), &j)); } +/*! +@brief throws ExceptionType with a fixed message; kept out of line so that callers do not carry the exception construction +*/ +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_literal(const exception_id id_, const char* what_arg, BasicJsonContext context) +{ + static_cast(id_); // unused when JSON_NOEXCEPTION is defined + static_cast(what_arg); + static_cast(context); + JSON_THROW(ExceptionType::create(id_, what_arg, context)); +} + +// JSON Patch errors; constructed out of line so that patch_inplace() does not carry them + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_add_parent(const BasicJsonType& parent) +{ + static_cast(parent); + JSON_THROW(out_of_range::create(exception_id::patch_add_parent_not_container, concat("cannot add value: the JSON Patch 'add' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_remove_parent(const BasicJsonType& parent) +{ + static_cast(parent); + JSON_THROW(out_of_range::create(exception_id::patch_remove_parent_not_container, concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_not_an_array(const BasicJsonType& j) +{ + static_cast(j); + JSON_THROW(parse_error::create(exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &j)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_missing_member(const StringType& op, const StringType& member, const bool string_type, const BasicJsonType& val) +{ + static_cast(op); + static_cast(member); + static_cast(string_type); + static_cast(val); + // context-sensitive error message + const std::string error_msg = (op == "op") ? std::string("operation") : concat("operation '", op, '\''); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat(error_msg, string_type ? " must have string member '" : " must have member '", member, "'"), &val)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_move_into_child(const StringType& from_path, const StringType& path, const BasicJsonType& result) +{ + static_cast(from_path); + static_cast(path); + static_cast(result); + JSON_THROW(out_of_range::create(exception_id::patch_move_into_child, concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_test_failed(const BasicJsonType& val) +{ + static_cast(val); + JSON_THROW(other_error::create(exception_id::patch_test_failed, concat("unsuccessful: ", val.dump()), &val)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_invalid_operation(const StringType& op, const BasicJsonType& val) +{ + static_cast(op); + static_cast(val); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat("operation value '", op, "' is invalid"), &val)); +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -5782,7 +5956,11 @@ void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) { case value_t::number_unsigned: { - val = static_cast(*j.template get_ptr()); + // never null: a basic_json whose number_unsigned_t is number_integer_t + // stores no number_unsigned values (#2573) + const auto* ptr = j.template get_ptr(); + JSON_ASSERT(ptr != nullptr); + val = static_cast(*ptr); break; } case value_t::number_integer: @@ -5852,7 +6030,7 @@ inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_flo } template -inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +inline void from_json(const BasicJsonType& j, number_unsigned_overload_t& val) { get_arithmetic_value(j, val); } @@ -5968,6 +6146,22 @@ template void from_json_array_reserve(ConstructibleArrayType& /*arr*/, std::size_t /*size*/, priority_tag<0> /*unused*/) {} +// append through push_back() where available; containers without it (such as +// std::set) fall back to insert(end(), ...), which is what std::inserter did +template +auto from_json_array_append(ConstructibleArrayType& arr, ValueType&& value, priority_tag<1> /*unused*/) +-> decltype(arr.push_back(std::forward(value)), void()) +{ + arr.push_back(std::forward(value)); +} + +template +void from_json_array_append(ConstructibleArrayType& arr, ValueType&& value, priority_tag<0> /*unused*/) +{ + using std::end; + arr.insert(end(arr), std::forward(value)); +} + template::value, @@ -5977,17 +6171,15 @@ auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, p j.template get(), void()) { - using std::end; - ConstructibleArrayType ret; - from_json_array_reserve(ret, j.size(), priority_tag<1> {}); - std::transform(j.begin(), j.end(), - std::inserter(ret, end(ret)), [](const BasicJsonType & i) + const auto* inner_array = j.template get_ptr(); + from_json_array_reserve(ret, inner_array->size(), priority_tag<1> {}); + for (const auto& elem : *inner_array) { // get() returns *this, this won't call a from_json // method when value_type is BasicJsonType - return i.template get(); - }); + from_json_array_append(ret, elem.template get(), priority_tag<1> {}); + } arr = std::move(ret); } @@ -6073,6 +6265,23 @@ template inline void from_json_object_reserve(ConstructibleObjectType& /*obj*/, std::size_t /*size*/, priority_tag<0> /*unused*/) {} +// the keys of the source are sorted for the usual object types, so hinting the +// end makes every insertion constant-time (for containers that order their keys +// the same way); containers without emplace_hint() (such as ordered_map) fall +// back to emplace() +template +auto from_json_object_emplace(ConstructibleObjectType& obj, Key&& key, Value&& value, priority_tag<1> /*unused*/) +-> decltype(obj.emplace_hint(obj.end(), std::forward(key), std::forward(value)), void()) +{ + obj.emplace_hint(obj.end(), std::forward(key), std::forward(value)); +} + +template +void from_json_object_emplace(ConstructibleObjectType& obj, Key&& key, Value&& value, priority_tag<0> /*unused*/) +{ + obj.emplace(std::forward(key), std::forward(value)); +} + template::value, int> = 0> inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) @@ -6087,7 +6296,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) from_json_object_reserve(ret, inner_object->size(), priority_tag<1> {}); for (const auto& p : *inner_object) { - ret.emplace(p.first, p.second.template get()); + from_json_object_emplace(ret, p.first, p.second.template get(), priority_tag<1> {}); } obj = std::move(ret); } @@ -6108,7 +6317,11 @@ inline void from_json(const BasicJsonType& j, ArithmeticType& val) { case value_t::number_unsigned: { - val = static_cast(*j.template get_ptr()); + // never null: a basic_json whose number_unsigned_t is number_integer_t + // stores no number_unsigned values (#2573) + const auto* ptr = j.template get_ptr(); + JSON_ASSERT(ptr != nullptr); + val = static_cast(*ptr); break; } case value_t::number_integer: @@ -7121,6 +7334,17 @@ struct external_constructor j.replace_value(value_t::array, value); } + /// an rvalue container: move its elements + template + static void construct_moving(BasicJsonType& j, CompatibleArrayType& arr) + { + using std::begin; + using std::end; + + const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(arr)), std::make_move_iterator(end(arr)))); + j.replace_value(value_t::array, value); + } + template static void construct(BasicJsonType& j, const std::vector& arr) { @@ -7171,6 +7395,17 @@ struct external_constructor j.replace_value(value_t::object, value); } + /// an rvalue container: move its values + template + static void construct_moving(BasicJsonType& j, CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(obj)), std::make_move_iterator(end(obj)))); + j.replace_value(value_t::object, value); + } + template < typename BasicJsonType, typename CompatibleObjectType, enable_if_t < !std::is_same::value, int > = 0 > static void construct(BasicJsonType& j, const CompatibleObjectType& obj) @@ -7257,14 +7492,14 @@ inline void to_json(BasicJsonType& j, FloatType val) noexcept } template::value, int> = 0> + enable_if_t, CompatibleNumberUnsignedType>::value, int> = 0> inline void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept { external_constructor::construct(j, static_cast(val)); } template::value, int> = 0> + enable_if_t::value, int> = 0> inline void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept { external_constructor::construct(j, static_cast(val)); @@ -7308,6 +7543,66 @@ inline void to_json(BasicJsonType& j, const CompatibleArrayType& arr) external_constructor::construct(j, arr); } +// types that have a dedicated to_json overload and must not be treated as generic containers +template +struct has_dedicated_to_json : std::false_type {}; + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template<> +struct has_dedicated_to_json : std::true_type {}; +#endif + +#if defined(__cpp_lib_char8_t) +template +struct has_dedicated_to_json> : std::true_type {}; +#endif + +#if JSON_HAS_RANGE_VIEW_CONVERSION +template +struct is_range_view_type : is_compatible_range_view {}; +#else +template +struct is_range_view_type : std::false_type {}; +#endif + +#if JSON_USE_OBJECTS_FOR_ENUM_KEYED_MAPS +template +struct is_enum_keyed_map_type : is_enum_keyed_map {}; +#else +template +struct is_enum_keyed_map_type : std::false_type {}; +#endif + +// rvalue (non-const) container: the generic to_json takes its elements by move +template +using is_movable_container_rvalue = negation, + std::is_const, + has_dedicated_to_json, + is_basic_json, + is_range_view_type, + is_enum_keyed_map_type, + std::is_same>>>; + +// rvalue array-like containers +template +using is_rvalue_array_to_json = conjunction, + is_compatible_array_type, + negation, + std::is_same, + is_compatible_object_type, + is_compatible_string_type, + is_compatible_binary_type>>>; + +// rvalue containers: their elements are moved instead of copied +template::value, int> = 0> +inline void to_json(BasicJsonType& j, CompatibleArrayType && arr) +{ + external_constructor::construct_moving(j, arr); +} + #if JSON_HAS_RANGE_VIEW_CONVERSION template < typename BasicJsonType, typename T, enable_if_t < is_compatible_range_view>::value @@ -7381,6 +7676,19 @@ inline void to_json(BasicJsonType& j, const EnumKeyedMap& map) } #endif +// rvalue object-like containers +template +using is_rvalue_object_to_json = conjunction, + negation>, + is_compatible_object_type>; + +template::value, int> = 0> +inline void to_json(BasicJsonType& j, CompatibleObjectType && obj) +{ + external_constructor::construct_moving(j, obj); +} + template inline void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) { @@ -7545,7 +7853,7 @@ struct adl_serializer }; NLOHMANN_JSON_NAMESPACE_END -// IWYU pragma: keep +// IWYU pragma: export // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -7653,7 +7961,7 @@ class byte_container_with_subtype : public BinaryType NLOHMANN_JSON_NAMESPACE_END // #include - +// IWYU pragma: export // #include // IWYU pragma: keep // #include @@ -7975,12 +8283,12 @@ NLOHMANN_JSON_NAMESPACE_END -#include // min +#include // count, find, min #include // array #include // size_t #include // uint32_t #include // strlen -#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer @@ -8269,6 +8577,7 @@ class iterator_input_adapter return result; } + ++eof_reads; return char_traits::eof(); } @@ -8278,6 +8587,26 @@ class iterator_input_adapter return static_cast(std::distance(begin, current)); } + // number of get_character() calls so far, including those that returned + // EOF; the lexer derives its position from it (see lexer::lazy_position) + std::size_t get_read_count() const + { + return get_consumed_count() + eof_reads; + } + + // number of newlines in the consumed range [0, last_index); @a line_start + // is set to the index after the last of them (0 if there is none); only + // valid when supports_seek + std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const + { + const auto to = std::next(begin, static_cast::difference_type>(last_index)); + const auto newline = char_traits::to_char_type('\n'); + using reverse_it = std::reverse_iterator; + const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline); + line_start = static_cast(std::distance(begin, last_newline.base())); + return static_cast(std::count(begin, last_newline.base(), newline)); + } + // append the already-consumed characters in the half-open range // [first_index, last_index) to @a out; only valid when supports_seek template @@ -8394,6 +8723,8 @@ class iterator_input_adapter IteratorType begin; IteratorType current; SentinelType end; + /// number of get_character() calls that returned EOF + std::size_t eof_reads = 0; template friend struct wide_string_input_helper; @@ -8914,30 +9245,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // move #include // vector -// #include - -// #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2021 The fast_float authors -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // array -#include // FLT_EVAL_METHOD -#include // localeconv -#include // size_t -#include // int64_t, uint64_t -#include // strtof, strtod, strtold -#include // memcpy -#include // numeric_limits -#include // string - // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -8950,12 +9257,13 @@ NLOHMANN_JSON_NAMESPACE_END #include // uint64_t -#if !defined(__SIZEOF_INT128__) && defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) - #include // __umulh, _umul128 +#include // memcpy +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) && (!defined(__SIZEOF_INT128__) || (!defined(__GNUC__) && !defined(__clang__))) + #include // __umulh, _umul128, _BitScanForward64, _BitScanReverse64 #endif -// #include - +// #include +// JSON_HEDLEY_ALWAYS_INLINE, NLOHMANN_JSON_NAMESPACE_BEGIN // Portable bit-level helpers for the number and string scanners. They use // compiler builtins or platform-specific intrinsics where available and plain @@ -8970,6 +9278,10 @@ inline int count_leading_zeros(std::uint64_t x) noexcept { #if defined(__GNUC__) || defined(__clang__) return __builtin_clzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanReverse64(&index, x); + return 63 - static_cast(index); #else int n = 0; for (int shift = 32; shift != 0; shift >>= 1) @@ -8984,6 +9296,29 @@ inline int count_leading_zeros(std::uint64_t x) noexcept #endif } +/// number of trailing zero bits of x (x != 0) +inline int count_trailing_zeros(std::uint64_t x) noexcept +{ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_ctzll(x); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long index = 0; // NOLINT(runtime/int): the type _BitScan*64 takes + _BitScanForward64(&index, x); + return static_cast(index); +#else + int n = 0; + for (int shift = 32; shift != 0; shift >>= 1) + { + if ((x << (64 - shift)) == 0) + { + n += shift; + x >>= shift; + } + } + return n; +#endif +} + /// the 128-bit product of two 64-bit numbers struct uint128_parts { @@ -9017,20 +9352,61 @@ inline uint128_parts full_multiplication(std::uint64_t a, std::uint64_t b) noexc #endif } -/// eight bytes as a little-endian word (compilers fold this into one load on -/// little-endian targets) -inline std::uint64_t read_eight_bytes(const char* p) noexcept +/// eight bytes as a little-endian word (a single load on little-endian +/// targets; always inlined, as GCC otherwise calls it in the number loops) +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const unsigned char* b) noexcept { - const auto* b = reinterpret_cast(p); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + // the byte order already matches (all MSVC targets are little-endian) + std::uint64_t result = 0; + std::memcpy(&result, b, sizeof(result)); + return result; +#else return static_cast(b[0]) | (static_cast(b[1]) << 8u) | (static_cast(b[2]) << 16u) | (static_cast(b[3]) << 24u) | (static_cast(b[4]) << 32u) | (static_cast(b[5]) << 40u) | (static_cast(b[6]) << 48u) | (static_cast(b[7]) << 56u); +#endif +} + +/// eight bytes as a little-endian word +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t read_eight_bytes(const char* p) noexcept +{ + return read_eight_bytes(reinterpret_cast(p)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) } } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2021 The fast_float authors +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // any_of +#include // array +#include // FLT_EVAL_METHOD +#include // localeconv +#include // size_t +#include // int64_t, uint64_t +#include // strtof, strtod, strtold +#include // memcpy +#include // numeric_limits +#include // string +#include // conditional, integral_constant, is_signed, true_type, false_type +#include // move + +// #include + // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -9057,6 +9433,10 @@ namespace detail constexpr std::int64_t pow5_128_smallest_power = -342; constexpr std::int64_t pow5_128_largest_power = 308; +// every entry of pow5_128() holds two 64-bit halves of 5^q, one per covered power of 5 +static_assert((pow5_128_largest_power - pow5_128_smallest_power + 1) * 2 == 1302, + "pow5_128_smallest_power/pow5_128_largest_power must match the size of the pow5_128() table"); + /*! @brief 128-bit approximations of 5^q for q in [-342, 308] @@ -9420,15 +9800,32 @@ NLOHMANN_JSON_NAMESPACE_END #endif // This file contains the value-conversion helpers used by the lexer to turn an -// already-validated number token into a value, without the locale/errno -// overhead of std::strtoull/std::strtod where possible. They are free functions -// so the lexer stays focused on scanning (see lexer::convert_number()) and so -// that other parsers of JSON text can convert tokens exactly like it does. +// already-validated number token into a value. Integers and binary32/binary64 +// floats (float, double, and long double where it is binary64) are converted +// by the library itself, without the locale/errno overhead of +// std::strtoull/std::strtod and correctly rounded; other long double formats +// use std::from_chars or std::strtold. They are free functions so the lexer +// stays focused on scanning (see lexer::convert_number()) and so that other +// parsers of JSON text can convert tokens exactly like it does. NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { +// whether an integer of a signed type is negative; the overload for other types +// avoids comparing an unsigned value with zero +template +constexpr bool is_negative_integer(NumberType x, std::true_type /*is_signed*/) noexcept +{ + return x < 0; +} + +template +constexpr bool is_negative_integer(NumberType /*unused*/, std::false_type /*is_signed*/) noexcept +{ + return false; +} + /*! @brief fast integer parser for an already-validated unsigned integer @@ -9462,8 +9859,10 @@ bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedT x = (x * 10u) + digit; } value = static_cast(x); - // reject values that do not round-trip into a narrower NumberUnsignedType - return static_cast(value) == x; + // reject values that do not round-trip into a narrower NumberUnsignedType, + // and values that wrap around to negative numbers if it is a signed type + // (number_unsigned_t may be the same type as number_integer_t, #2573) + return static_cast(value) == x && !is_negative_integer(value, std::is_signed()); } /*! @@ -9500,233 +9899,291 @@ bool parse_integer_signed(const char* first, const char* last, NumberIntegerType } /*! -@brief exact fast path for parsing a `double` (Clinger's algorithm) - -For the common case - at most 19 significant digits, a decimal exponent in -[-22, 22], and a significand below 2^53 - the value equals significand * -10^exp computed in IEEE-754 double arithmetic, which is exact under -round-to-nearest because both operands are exactly representable. This is the -same fast path used by fast_float/simdjson; the general cases are left to -std::strtod. The parser only activates for number_float_t == double; float and -long double keep the std::strtof/std::strtold paths (see the templated overload -below). - -@param[in] first pointer to the first character of the number -@param[in] last pointer past the last character -@param[out] out the parsed value on success -@return true if the value was parsed exactly; false to fall back to strtod +@brief parameters of the IEEE-754 binary32 and binary64 formats for the float + conversion (after fast_float's binary_format) */ -inline bool parse_float_fast(const char* first, const char* last, double& out) noexcept +template +struct ieee_binary_format; + +template<> +struct ieee_binary_format<24> // binary32 { -#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 - // Clinger's fast path is only exact when double operations are evaluated in - // true double precision. On platforms that keep intermediates in extended - // precision (e.g. the x87 FPU on 32-bit x86, where FLT_EVAL_METHOD == 2) the - // single significand * 10^scale step is double-rounded and can be 1 ULP off, - // so decline and let the caller fall back to the correctly-rounded - // std::from_chars / std::strtod path. - static_cast(first); - static_cast(last); - static_cast(out); - return false; -#else - static const std::array powers_of_ten = + static constexpr int mantissa_bits() noexcept { - { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 - } - }; + return 23; + } + static constexpr int sign_bit() noexcept + { + return 31; + } + static constexpr int minimum_exponent() noexcept + { + return -127; + } + static constexpr int infinite_power() noexcept + { + return 0xFF; + } + // w * 10^q with w < 2^64 is below half the smallest subnormal number for + // q < smallest_power_of_ten() and at least infinity for q > largest_power_of_ten() + static constexpr int smallest_power_of_ten() noexcept + { + return -64; + } + static constexpr int largest_power_of_ten() noexcept + { + return 38; + } + // w * 10^q can only be exactly between two numbers for q in this range + static constexpr int min_exponent_round_to_even() noexcept + { + return -17; + } + static constexpr int max_exponent_round_to_even() noexcept + { + return 10; + } + // Clinger's fast path: w and 10^|q| are exact + static constexpr int max_exponent_fast_path() noexcept + { + return 10; + } + static constexpr std::uint64_t max_mantissa_fast_path() noexcept + { + return std::uint64_t{2} << 23u; + } + // a midpoint between two numbers has at most this many significant digits + static constexpr std::int64_t max_digits() noexcept + { + return 114; + } +}; - const char* p = first; - bool negative = false; - if (p != last && (*p == '-' || *p == '+')) - { - negative = (*p == '-'); - ++p; - } - - std::uint64_t significand = 0; - int num_digits = 0; - int fractional_digits = 0; - bool seen_dot = false; - bool any_digit = false; - for (; p != last; ++p) - { - const char c = *p; - if (c >= '0' && c <= '9') - { - any_digit = true; - if (JSON_HEDLEY_UNLIKELY(num_digits >= 19)) - { - return false; // significand may not fit into uint64_t - } - significand = (significand * 10u) + static_cast(c - '0'); - ++num_digits; - fractional_digits += static_cast(seen_dot); - } - else if (c == '.') - { - if (JSON_HEDLEY_UNLIKELY(seen_dot)) - { - return false; - } - seen_dot = true; - } - else if (c == 'e' || c == 'E') - { - ++p; - break; - } - else - { - return false; - } - } - if (JSON_HEDLEY_UNLIKELY(!any_digit)) - { - return false; - } - - int exponent = 0; - if (p != last) // an exponent part remains - { - bool exp_negative = false; - if (p != last && (*p == '-' || *p == '+')) - { - exp_negative = (*p == '-'); - ++p; - } - bool any_exp_digit = false; - for (; p != last; ++p) - { - if (JSON_HEDLEY_UNLIKELY(*p < '0' || *p > '9')) - { - return false; - } - exponent = (exponent * 10) + (*p - '0'); - any_exp_digit = true; - if (JSON_HEDLEY_UNLIKELY(exponent > 9999)) - { - return false; - } - } - if (JSON_HEDLEY_UNLIKELY(!any_exp_digit)) - { - return false; - } - if (exp_negative) - { - exponent = -exponent; - } - } - - const int scale = exponent - fractional_digits; - if (JSON_HEDLEY_UNLIKELY(significand >= (static_cast(1) << 53))) - { - return false; // significand not exactly representable as double - } - - auto result = static_cast(significand); - if (scale >= 0) - { - if (JSON_HEDLEY_UNLIKELY(scale > 22)) - { - return false; - } - result *= powers_of_ten[static_cast(scale)]; - } - else - { - if (JSON_HEDLEY_UNLIKELY(-scale > 22)) - { - return false; - } - result /= powers_of_ten[static_cast(-scale)]; - } - out = negative ? -result : result; - return true; -#endif -} - -/// fast float path is only exact for `double`; decline for float/long double -template -bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept +template<> +struct ieee_binary_format<53> // binary64 { - return false; -} + static constexpr int mantissa_bits() noexcept + { + return 52; + } + static constexpr int sign_bit() noexcept + { + return 63; + } + static constexpr int minimum_exponent() noexcept + { + return -1023; + } + static constexpr int infinite_power() noexcept + { + return 0x7FF; + } + static constexpr int smallest_power_of_ten() noexcept + { + return -342; + } + static constexpr int largest_power_of_ten() noexcept + { + return 308; + } + static constexpr int min_exponent_round_to_even() noexcept + { + return -4; + } + static constexpr int max_exponent_round_to_even() noexcept + { + return 23; + } + static constexpr int max_exponent_fast_path() noexcept + { + return 22; + } + static constexpr std::uint64_t max_mantissa_fast_path() noexcept + { + return std::uint64_t{2} << 52u; + } + static constexpr std::int64_t max_digits() noexcept + { + return 769; + } +}; /*! -@brief parse a float with std::from_chars (Eisel-Lemire) when available +@brief whether @a FloatType is IEEE-754 binary32 or binary64 -std::from_chars is locale-independent, correctly rounded, and - via the -Eisel-Lemire algorithm in modern standard libraries - much faster than strtod -over the whole value range (not just the Clinger subset). It is used only when -__cpp_lib_to_chars indicates full floating-point support and only when it -consumes the entire token ([first, last)). An under-/overflow (result_out_of_range) also declines, so -the caller's strtod fallback supplies the well-defined ±inf/0 result the parser -expects (side-stepping the P4168 divergence between implementations). - -@return true if the value was parsed exactly and fully; false to fall back +These formats (float, double, and long double where it is binary64, e.g. +with MSVC or on Apple arm64) are converted by parse_float_native(). The +predicate is the one the serializer uses to choose Grisu2. */ template -bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +struct has_native_float_format { - // JSON_HAS_CPP_17 must gate the use as well as the include above: - // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even - // in C++14 mode, where is not included. -#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) - const auto result = std::from_chars(first, last, out); - return result.ec == std::errc() && result.ptr == last; -#else - static_cast(first); - static_cast(last); - static_cast(out); - return false; -#endif -} + static constexpr bool value = + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 24 && std::numeric_limits::max_exponent == 128) || + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && std::numeric_limits::max_exponent == 1024); +}; -/// whether the eight bytes of @a v (see read_eight_bytes()) are ASCII digits -/// (after fast_float's is_made_of_eight_digits_fast) -inline bool is_eight_digits(std::uint64_t v) noexcept -{ - return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u; -} +/// the C++ type (float or double) that holds a binary32 or binary64 @a FloatType +template +using native_float_t = typename std::conditional::digits == 24, float, double>::type; /// the value of the eight ASCII digits in @a v (see read_eight_bytes()), three -/// multiplications instead of eight (after simdjson and fast_float) -inline std::uint32_t parse_eight_digits(std::uint64_t v) noexcept +/// multiplications instead of eight (after simdjson and fast_float); always +/// inlined, as GCC otherwise calls it in the number loops +JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noexcept { v = ((v & 0x0F0F0F0F0F0F0F0Fu) * 2561u) >> 8u; v = ((v & 0x00FF00FF00FF00FFu) * 6553601u) >> 16u; return static_cast(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u); } +/// whether [first, last) contains a digit other than '0' +inline bool has_nonzero_digit(const char* first, const char* last) noexcept +{ + return std::any_of(first, last, [](char c) noexcept + { + return c != '0'; + }); +} + +/// the value of the validated exponent digits [+-]?[0-9]+ in [first, last), +/// saturated far beyond every range +inline std::int64_t parse_float_exponent(const char* first, const char* last) noexcept +{ + const bool negative = *first == '-'; + first += (*first == '-' || *first == '+') ? 1 : 0; + constexpr std::int64_t saturation = 100000000000000000; // 10^17 + std::int64_t value = 0; + for (; first != last; ++first) + { + if (value < saturation) + { + value = (value * 10) + (*first - '0'); + } + } + return negative ? -value : value; +} + +/// a float token as w * 10^exponent, see parse_float_significand() +struct float_significand +{ + std::uint64_t w = 0; ///< the first (at most 19) significant digits + std::int64_t exponent = 0; ///< the decimal exponent of the last digit in w + bool negative = false; ///< whether the token starts with '-' + bool truncated = false; ///< whether nonzero digits follow the ones in w +}; + /*! -@brief the double nearest to w * 10^q (Eisel-Lemire) +@brief split a validated number token into sign, significand, and exponent + +The lexer has validated the token against the JSON grammar and knows where its +parts are, so this needs no character classification: the integer part ends at +@a decimal_point_position (or @a mantissa_end), the fraction at @a mantissa_end, +and an exponent follows. At most 19 significant digits are kept; the value then +lies in [w, w + 1) * 10^exponent, and is exactly w * 10^exponent unless +truncated is set. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +*/ +inline float_significand parse_float_significand(const char* first, const char* last, + std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +{ + float_significand s; + const char* p = first; + s.negative = *p == '-'; + p += s.negative ? 1 : 0; + const bool has_dot = decimal_point_position != std::string::npos; + const char* const mantissa_last = first + mantissa_end; + const char* const integer_last = has_dot ? first + decimal_point_position : mantissa_last; + + std::uint64_t w = 0; + int remaining = 19; // digits that still fit into w + if (*p != '0') // the integer part is "0" or [1-9][0-9]* + { + while (remaining >= 8 && integer_last - p >= 8) + { + w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p)); + p += 8; + remaining -= 8; + } + for (; remaining > 0 && p != integer_last; ++p, --remaining) + { + w = (w * 10u) + static_cast(*p - '0'); + } + s.exponent = integer_last - p; + s.truncated = has_nonzero_digit(p, integer_last); + } + + if (has_dot) + { + p = integer_last + 1; + if (w == 0) + { + // zeros after the decimal point of "0." are not significant + const char* const zeros = p; + while (p != mantissa_last && *p == '0') + { + ++p; + } + s.exponent -= p - zeros; + } + const char* const digits = p; + while (remaining >= 8 && mantissa_last - p >= 8) + { + w = (w * 100000000u) + parse_eight_digits(read_eight_bytes(p)); + p += 8; + remaining -= 8; + } + for (; remaining > 0 && p != mantissa_last; ++p, --remaining) + { + w = (w * 10u) + static_cast(*p - '0'); + } + s.exponent -= p - digits; + s.truncated = s.truncated || has_nonzero_digit(p, mantissa_last); + } + + if (mantissa_last != last) + { + s.exponent += parse_float_exponent(mantissa_last + 1, last); + } + s.w = w; + return s; +} + +/*! +@brief the bits of the float nearest to w * 10^q (Eisel-Lemire) The algorithm of Daniel Lemire, "Number Parsing at a Gigabyte per Second" (Software: Practice and Experience, 2021), after fast_float's compute_float (used under the MIT license). With a 128-bit approximation of 5^q, the product -is always sufficient to round correctly for w with at most 19 digits (Noble -Mushtak and Daniel Lemire, "Fast number parsing without fallback", Software: -Practice and Experience, 2023). Only integer arithmetic is used, so the result -does not depend on the floating-point environment. +is always sufficient to round correctly for w < 2^64 (Noble Mushtak and Daniel +Lemire, "Fast number parsing without fallback", Software: Practice and +Experience, 2023). Only integer arithmetic is used, so the result does not +depend on the floating-point environment. +It is always inlined, like decimal_to_float(), so that hot loops of callers +keep the whole conversion inline. + +@tparam Format ieee_binary_format<24> (binary32) or ieee_binary_format<53> (binary64) @param[in] q decimal exponent -@param[in] w significand, w != 0 +@param[in] w significand @return the IEEE-754 bits of the positive result (0 for underflow, infinity for overflow) */ -inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept +template +JSON_HEDLEY_ALWAYS_INLINE std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept { - constexpr int mantissa_bits = 52; - constexpr std::uint64_t infinity = std::uint64_t{0x7FF} << mantissa_bits; - if (q < pow5_128_smallest_power) + constexpr int mantissa_bits = Format::mantissa_bits(); + constexpr std::uint64_t infinity = static_cast(Format::infinite_power()) << mantissa_bits; + if (w == 0 || q < Format::smallest_power_of_ten()) { return 0; } - if (q > pow5_128_largest_power) + if (q > Format::largest_power_of_ten()) { return infinity; } @@ -9750,8 +10207,8 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept const auto upperbit = static_cast(product.high >> 63u); const int shift = upperbit + 64 - mantissa_bits - 3; std::uint64_t mantissa = product.high >> static_cast(shift); - // floor(log2(10^q)) + 63 + 1023, with log2(10) ~ 217706 / 2^16 - std::int64_t power2 = (((152170 + 65536) * q) >> 16) + 63 + upperbit - lz + 1023; + // floor(log2(10^q)) + 63 + bias, with log2(10) ~ 217706 / 2^16 + std::int64_t power2 = (((152170 + 65536) * q) >> 16) + 63 + upperbit - lz - Format::minimum_exponent(); if (power2 <= 0) // subnormal { @@ -9760,17 +10217,18 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept return 0; } mantissa >>= static_cast(-power2 + 1); + // no tie is possible here: that needs a small |q| mantissa += (mantissa & 1u); mantissa >>= 1u; // rounding up may produce the smallest normal number power2 = (mantissa < (std::uint64_t{1} << mantissa_bits)) ? 0 : 1; - return mantissa | (static_cast(power2) << mantissa_bits); + return (mantissa & ((std::uint64_t{1} << mantissa_bits) - 1)) | (static_cast(power2) << mantissa_bits); } - // a value exactly between two doubles rounds to even; this can only + // a value exactly between two floats rounds to even; this can only // happen for small |q|, where 5^q is exact - if (product.low <= 1 && q >= -4 && q <= 23 && (mantissa & 3u) == 1 - && (mantissa << static_cast(shift)) == product.high) + if (product.low <= 1 && q >= Format::min_exponent_round_to_even() && q <= Format::max_exponent_round_to_even() + && (mantissa & 3u) == 1 && (mantissa << static_cast(shift)) == product.high) { mantissa &= ~std::uint64_t{1}; } @@ -9782,198 +10240,420 @@ inline std::uint64_t eisel_lemire(std::int64_t q, std::uint64_t w) noexcept ++power2; } mantissa &= ~(std::uint64_t{1} << mantissa_bits); - if (power2 >= 0x7FF) + if (power2 >= Format::infinite_power()) { return infinity; } return mantissa | (static_cast(power2) << mantissa_bits); } -/*! -@brief parse a validated float token with the Eisel-Lemire algorithm - -The significand is accumulated eight digits at a time where possible. A token -with more than 19 significant digits is truncated to w; the value then lies -in [w, w + 1) * 10^q, and it is only returned if both ends round to the same -double, which covers all but a few such tokens. - -@param[in] first pointer to the first character of the token -@param[in] last pointer past the last character -@param[out] out the correctly rounded value on success (±infinity if it - overflows, like strtod) -@return true on success; false if strtod must decide -*/ -inline bool parse_float_eisel_lemire(const char* first, const char* last, double& out) noexcept +/// an unsigned integer of up to 4096 bits for digit_comparison() (32-bit limbs, +/// so only 32x32->64-bit multiplications are needed) +class float_bigint { - const char* p = first; - const bool negative = (p != last && *p == '-'); - if (negative) + public: + explicit float_bigint(std::uint64_t value) noexcept { - ++p; + for (; value != 0; value >>= 32u) + { + limbs[count++] = static_cast(value); + } } - std::uint64_t w = 0; - unsigned int digits = 0; // significant digits in w - std::int64_t exponent = 0; - bool truncated = false; - bool in_fraction = false; - for (;;) + /// *this = *this * factor + summand + void multiply_add(std::uint32_t factor, std::uint32_t summand) noexcept { - // eight digits at a time, as long as they fit into w - while (w != 0 && digits <= 19u - 8u && last - p >= 8) + std::uint64_t carry = summand; + for (std::size_t i = 0; i < count; ++i) { - const std::uint64_t v = read_eight_bytes(p); - if (!is_eight_digits(v)) - { - break; - } - w = (w * 100000000u) + parse_eight_digits(v); - digits += 8u; - exponent -= in_fraction ? 8 : 0; - p += 8; + const std::uint64_t product = (static_cast(limbs[i]) * factor) + carry; + limbs[i] = static_cast(product); + carry = product >> 32u; } - if (p == last) + if (carry != 0) { - break; + JSON_ASSERT(count < limbs.size()); + limbs[count++] = static_cast(carry); } - const char c = *p; - if (c >= '0' && c <= '9') + } + + /// *this = *this * 5^n + void multiply_power_of_five(std::int64_t n) noexcept + { + static const std::array powers = { - if (w == 0 && c == '0') - { - // leading zeros are not significant, but scale a fraction - exponent -= in_fraction ? 1 : 0; - } - else if (digits < 19u) - { - w = (w * 10u) + static_cast(c - '0'); - ++digits; - exponent -= in_fraction ? 1 : 0; - } - else - { - // dropped: the value lies between w and w + 1 (in units of - // the last kept digit) unless all dropped digits are zero - truncated = truncated || c != '0'; - exponent += in_fraction ? 0 : 1; - } - ++p; + {1u, 5u, 25u, 125u, 625u, 3125u, 15625u, 78125u, 390625u, 1953125u, 9765625u, 48828125u, 244140625u, 1220703125u} + }; + for (; n >= 13; n -= 13) + { + multiply_add(powers[13], 0); } - else if (c == '.') + multiply_add(powers[static_cast(n)], 0); + } + + /// *this = *this * 2^n + void shift_left(std::int64_t n) noexcept + { + if (count == 0) { - in_fraction = true; - ++p; + return; + } + const auto limb_shift = static_cast(n / 32); + const auto bit_shift = static_cast(n % 32); + JSON_ASSERT(count + limb_shift + 1 <= limbs.size()); + if (bit_shift != 0) + { + std::uint32_t carry = 0; + for (std::size_t i = 0; i < count; ++i) + { + const std::uint32_t limb = limbs[i]; + limbs[i] = (limb << bit_shift) | carry; + carry = limb >> (32u - bit_shift); + } + if (carry != 0) + { + limbs[count++] = carry; + } + } + if (limb_shift != 0) + { + for (std::size_t i = count; i-- > 0;) + { + limbs[i + limb_shift] = limbs[i]; + } + for (std::size_t i = 0; i < limb_shift; ++i) + { + limbs[i] = 0; + } + count += limb_shift; + } + } + + /// -1, 0, or 1 if *this is less than, equal to, or greater than @a other + int compare(const float_bigint& other) const noexcept + { + if (count != other.count) + { + return count < other.count ? -1 : 1; + } + for (std::size_t i = count; i-- > 0;) + { + if (limbs[i] != other.limbs[i]) + { + return limbs[i] < other.limbs[i] ? -1 : 1; + } + } + return 0; + } + + private: + std::array limbs{{}}; + std::size_t count = 0; +}; + +/*! +@brief round a token exactly when eisel_lemire() cannot decide (slow path) + +The value v of the token lies strictly between two adjacent floats, whose +lower one has the bits @a lower, and the result depends on whether v is below, +at, or above the midpoint m between them. Both are compared exactly as big +integers: v = D * 10^s with the significant digits D (at most +Format::max_digits() of them, more than any midpoint has; further nonzero +digits only put v above m) and m = (2 * mantissa + 1) * 2^(e - 1). This is the +digit comparison of fast_float (Daniel Lemire and contributors, used under the +MIT license), simplified by starting from the two candidates. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] lower the bits of the float below v +@return the bits of the correctly rounded result +*/ +template +std::uint64_t digit_comparison(const char* first, const char* last, std::uint64_t lower) noexcept +{ + const char* p = first + ((*first == '-') ? 1 : 0); + + // D, in chunks of up to 9 digits, and s + float_bigint digits(0); + std::int64_t count = 0; + std::int64_t point = 0; // the value is 0.D... * 10^point + bool truncated = false; + std::uint32_t chunk = 0; + int chunk_digits = 0; + static const std::array powers_of_ten = {{1u, 10u, 100u, 1000u, 10000u, 100000u, 1000000u, 10000000u, 100000000u, 1000000000u}}; + const auto append = [&](char c) noexcept + { + if (count < Format::max_digits()) + { + chunk = (chunk * 10u) + static_cast(c - '0'); + ++count; + if (++chunk_digits == 9) + { + digits.multiply_add(powers_of_ten[9], chunk); + chunk = 0; + chunk_digits = 0; + } } else { - break; // 'e' or 'E' + truncated = truncated || c != '0'; + } + }; + bool significant = false; + for (; p != last && *p >= '0' && *p <= '9'; ++p) + { + significant = significant || *p != '0'; + if (significant) + { + append(*p); + ++point; } } - - if (p != last) + if (p != last && *p == '.') { - ++p; // 'e' or 'E' - bool exp_negative = false; - if (p != last && (*p == '-' || *p == '+')) + for (++p; p != last && *p >= '0' && *p <= '9'; ++p) { - exp_negative = (*p == '-'); - ++p; - } - std::int64_t exp_value = 0; - for (; p != last; ++p) - { - // saturate: any exponent beyond this under- or overflows anyway - if (exp_value < 100000) + significant = significant || *p != '0'; + if (significant) { - exp_value = (exp_value * 10) + (*p - '0'); + append(*p); + } + else + { + --point; } } - exponent += exp_negative ? -exp_value : exp_value; } - - std::uint64_t bits = 0; - if (w != 0) + if (chunk_digits != 0) { - bits = eisel_lemire(exponent, w); - if (truncated && (w + 1 == 0 || eisel_lemire(exponent, w + 1) != bits)) - { - return false; - } + digits.multiply_add(powers_of_ten[static_cast(chunk_digits)], chunk); } - bits |= negative ? (std::uint64_t{1} << 63u) : 0u; - static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); - std::memcpy(&out, &bits, sizeof(out)); - return true; + if (p != last) + { + point += parse_float_exponent(p + 1, last); + } + const std::int64_t s = point - count; // v = D * 10^s + + // the midpoint above the lower candidate + constexpr int mantissa_bits = Format::mantissa_bits(); + const std::uint64_t exponent_field = lower >> mantissa_bits; + std::uint64_t mantissa = lower & ((std::uint64_t{1} << mantissa_bits) - 1); + std::int64_t e = 1 + Format::minimum_exponent() - mantissa_bits; // of the smallest subnormal number + if (exponent_field != 0) + { + mantissa |= std::uint64_t{1} << mantissa_bits; + e += static_cast(exponent_field) - 1; + } + float_bigint midpoint((2 * mantissa) + 1); + const std::int64_t midpoint_exponent = e - 1; // m = midpoint * 2^midpoint_exponent + + // compare D * 5^s * 2^s with midpoint * 2^midpoint_exponent + if (s >= 0) + { + digits.multiply_power_of_five(s); + } + else + { + midpoint.multiply_power_of_five(-s); + } + const std::int64_t shift = s - midpoint_exponent; + if (shift >= 0) + { + digits.shift_left(shift); + } + else + { + midpoint.shift_left(-shift); + } + const int order = digits.compare(midpoint); + const bool round_up = order > 0 || (order == 0 && (truncated || (mantissa & 1u) != 0)); + return lower + (round_up ? 1u : 0u); } -/// Eisel-Lemire is only implemented for `double` -template -bool parse_float_eisel_lemire(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept +/// the double with the IEEE-754 bits @a bits +inline void float_from_bits(std::uint64_t bits, double& value) noexcept { - return false; + static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits"); + std::memcpy(&value, &bits, sizeof(value)); +} + +/// the float with the IEEE-754 bits @a bits (the lower 32) +inline void float_from_bits(std::uint64_t bits, float& value) noexcept +{ + static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits"); + const auto bits32 = static_cast(bits); + std::memcpy(&value, &bits32, sizeof(bits32)); +} + +/// the powers of ten that are exact in binary64 (up to 10^22) +inline double exact_power_of_ten(std::int64_t n, double /*tag*/) noexcept +{ + static const std::array powers = + { + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22 + } + }; + return powers[static_cast(n)]; +} + +/// the powers of ten that are exact in binary32 (up to 10^10) +inline float exact_power_of_ten(std::int64_t n, float /*tag*/) noexcept +{ + static const std::array powers = + { + {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f, 1e7f, 1e8f, 1e9f, 1e10f} + }; + return powers[static_cast(n)]; } /*! -@brief check whether Clinger's fast path can still succeed for a float token +@brief the binary32/binary64 value of a significand that was not truncated -parse_float_fast() needs a significand below 2^53. A mantissa with 17 or -more significant digits is at least 10^16 and therefore always exceeds it, -so calling the fast path would walk the token one extra time only to -decline before strtod has to run anyway. +The result is (-1)^negative * w * 10^exponent, correctly rounded (ties to +even): with Clinger's fast path where w and 10^|exponent| are exact, so that a +single floating-point operation rounds (only where intermediate results are not +kept in extended precision, see FLT_EVAL_METHOD), and with eisel_lemire() +otherwise. A value too large for the type becomes ±infinity, a value too small +±0. -Significant digits are the mantissa's digits from the first nonzero one on; -the sign, the decimal point, leading zeros, and the exponent do not count. -The answer is derived from indices - the digits are not scanned again - so -this stays off the hot path of the number scanners. +This is the core of the conversion that other parsers of JSON text share: they +can split a token themselves and still get the lexer's result. It is always +inlined, so that their hot loops keep the whole conversion inline. -@param[in] token the validated number token ('.' as decimal point) -@param[in] decimal_point_position index of the '.' in @a token, or - std::string::npos if there is none -@param[in] mantissa_end offset just past the last mantissa byte -@return false if parse_float_fast() is guaranteed to decline +@param[in] s the significand, with s.truncated == false */ -inline bool mantissa_fits_clinger(const char* token, std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +template +JSON_HEDLEY_ALWAYS_INLINE FloatType decimal_to_float(const float_significand& s) noexcept { - // 10^16 already exceeds 2^53, so 17 digits can never fit - constexpr std::size_t limit = 17; + using result_type = native_float_t; + using format = ieee_binary_format::digits>; + JSON_ASSERT(!s.truncated); - const std::size_t neg = (token[0] == '-') ? 1u : 0u; - const std::size_t has_dot = (decimal_point_position != std::string::npos) ? 1u : 0u; - // the JSON grammar restricts the integer part to "0" or [1-9][0-9]*, so - // a leading zero can only be a lone "0", which is not significant - const std::size_t lead_zero = (token[neg] == '0') ? 1u : 0u; - JSON_ASSERT(mantissa_end >= neg + has_dot + lead_zero); - std::size_t digits = mantissa_end - neg - has_dot - lead_zero; - - if (JSON_HEDLEY_LIKELY(digits < limit)) +#if !defined(FLT_EVAL_METHOD) || FLT_EVAL_METHOD == 0 + if (s.exponent >= -format::max_exponent_fast_path() && s.exponent <= format::max_exponent_fast_path() + && s.w <= format::max_mantissa_fast_path()) { - return true; - } - - // Only a number below 1 can carry further insignificant zeros, and only - // while the count stays at the limit does removing them change the - // answer - so this loop is skipped for all but a few tokens. The - // fraction is located through decimal_point_position rather than by - // searching '.'. - if (lead_zero != 0) - { - JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit - for (std::size_t i = decimal_point_position + 1; - digits >= limit && i < mantissa_end && token[i] == '0'; ++i) + auto value = static_cast(s.w); + if (s.exponent < 0) { - --digits; + value /= exact_power_of_ten(-s.exponent, result_type{}); } + else + { + value *= exact_power_of_ten(s.exponent, result_type{}); + } + const FloatType result = s.negative ? -value : value; + return result; + } +#endif + + result_type value{}; + float_from_bits(eisel_lemire(s.exponent, s.w) | (s.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + const FloatType result = value; + return result; +} + +/*! +@brief convert a validated number token to the nearest binary32/binary64 value + +The conversion is correctly rounded (ties to even) and independent of the +locale and of the C and C++ libraries: +1. parse_float_significand() splits the token into w * 10^q. +2. If no digits were dropped, decimal_to_float() rounds w * 10^q (Clinger's + fast path or Eisel-Lemire). +3. Otherwise, the value lies in [w, w + 1) * 10^q: if eisel_lemire() rounds + both ends to the same value, so does the token (in all but rare cases). +4. Otherwise, digit_comparison() compares the token exactly with the midpoint + between the two candidates. +A value too large for the type becomes ±infinity (the parser reports +out_of_range.406), a value too small ±0. + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +*/ +template +FloatType parse_float_native(const char* first, const char* last, + std::size_t decimal_point_position, std::size_t mantissa_end) noexcept +{ + using result_type = native_float_t; + using format = ieee_binary_format::digits>; + static_assert(std::numeric_limits::digits == std::numeric_limits::digits, "unexpected float format"); + + const float_significand s = parse_float_significand(first, last, decimal_point_position, mantissa_end); + if (JSON_HEDLEY_LIKELY(!s.truncated)) + { + return decimal_to_float(s); } - return digits < limit; + std::uint64_t bits = eisel_lemire(s.exponent, s.w); + if (JSON_HEDLEY_UNLIKELY(bits != eisel_lemire(s.exponent, s.w + 1))) + { + bits = digit_comparison(first, last, bits); + } + result_type value{}; + float_from_bits(bits | (s.negative ? (std::uint64_t{1} << format::sign_bit()) : 0u), value); + const FloatType result = value; + return result; +} + +/*! +@brief parse a float with std::from_chars when available + +Only used for the formats parse_float_native() does not convert (long double +formats other than binary64). std::from_chars is locale-independent and +correctly rounded. It is used only when __cpp_lib_to_chars indicates full +floating-point support and only when it consumes the entire token ([first, +last)). An under-/overflow (result_out_of_range) also declines, so the +caller's strtold fallback supplies the well-defined ±inf/0 result the parser +expects (side-stepping the P4168 divergence between implementations). + +@return true if the value was parsed exactly and fully; false to fall back +*/ +template +bool parse_float_from_chars(const char* first, const char* last, FloatType& out) noexcept +{ + // JSON_HAS_CPP_17 must gate the use as well as the include above: + // some standard libraries (e.g. libstdc++ 15) define __cpp_lib_to_chars even + // in C++14 mode, where is not included. +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) + const auto result = std::from_chars(first, last, out); + return result.ec == std::errc() && result.ptr == last; +#else + static_cast(first); + static_cast(last); + static_cast(out); + return false; +#endif +} + +/// binary32 and binary64: the library's own conversion, which always succeeds +template +bool convert_float_fast(const char* first, const char* last, std::size_t decimal_point_position, + std::size_t mantissa_end, FloatType& value, std::true_type /*native*/) noexcept +{ + value = parse_float_native(first, last, decimal_point_position, mantissa_end); + return true; +} + +/// other formats (long double on x87, binary128, double-double): std::from_chars, if available +template +bool convert_float_fast(const char* first, const char* last, std::size_t /*decimal_point_position*/, + std::size_t /*mantissa_end*/, FloatType& value, std::false_type /*native*/) noexcept +{ + return parse_float_from_chars(first, last, value); } /*! @brief convert a validated float token without the C library, if possible -Tries std::from_chars (when available), Clinger's exact fast path (double -only, skipped when it cannot succeed), and the Eisel-Lemire algorithm (double -only). +float, double, and long double where it is binary64 are always converted, by +parse_float_native(). Other long double formats are converted with +std::from_chars where the standard library supports it. @param[in] first pointer to the first character of the token @param[in] last pointer past the last character @@ -9989,19 +10669,8 @@ template bool convert_float_fast(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end, FloatType& value) noexcept { - if (parse_float_from_chars(first, last, value)) - { - return true; - } - // Skipping a fast path that cannot succeed is lossless and saves a full - // extra pass over the token's bytes, which otherwise shows up on - // high-precision inputs such as canada.json - if (mantissa_fits_clinger(first, decimal_point_position, mantissa_end) - && parse_float_fast(first, last, value)) - { - return true; - } - return parse_float_eisel_lemire(first, last, value); + return convert_float_fast(first, last, decimal_point_position, mantissa_end, value, + std::integral_constant::value> {}); } /// std::strtof, std::strtod, or std::strtold, chosen by the type of @a f @@ -10025,36 +10694,45 @@ inline void strtof_by_type(long double& f, const char* str, char** endptr) noexc f = std::strtold(str, endptr); } -/// return the decimal point of the current locale -inline char get_decimal_point() noexcept +/// return the decimal point of the current locale (it may be longer than one byte) +inline std::string get_decimal_point() { const auto* loc = localeconv(); JSON_ASSERT(loc != nullptr); - return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + return (loc->decimal_point == nullptr || *loc->decimal_point == '\0') ? "." : loc->decimal_point; } /*! @brief convert a validated float token with strtof/strtod/strtold +Only used for what convert_float_fast() does not convert: long double formats +other than binary64 where std::from_chars is unavailable or reports an under- +or overflow, and floating-point types that are not IEEE-754 (see +has_native_float_format). + These functions expect the decimal point of the *current* locale, so it is looked up right before the conversion instead of once when the lexer is constructed: a locale change in between (by a parser callback, a SAX -handler, or another thread) must not truncate the value (#5198). The -token has been validated before, so if the conversion stops early and the -decimal point changed in the meantime, the locale changed between the +handler, or another thread) must not truncate the value (#5198). A +single-byte decimal point is substituted in place and restored afterwards, +because the token is also handed to the SAX interface. A longer one (e.g., +the two-byte U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) is put into a copy of the +token instead (#5660). + +The token has been validated before, so if the conversion stops early and +the decimal point changed in the meantime, the locale changed between the lookup and the call, and the conversion is repeated with the new decimal -point. If the decimal point did not change, a retry cannot succeed: the -locale's decimal point is not a single character (e.g., the two-byte -U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) and cannot be substituted in place. -The value strtod parsed up to that point is kept, as before this change. +point. If it did not change, the value strtod parsed up to that point is +kept. Note that changing the locale in another thread *while* strtod runs is undefined behavior of the C library, which this function cannot prevent. -@param[in,out] token the token with '.' as decimal point; its - decimal point is replaced during the - conversion and restored afterwards - (data() must be NUL-terminated) +@param[in,out] token the token with '.' as decimal point; a + single-byte decimal point is put in + place during the conversion and + restored afterwards (data() must be + NUL-terminated) @param[in] decimal_point_position index of the '.' in @a token, or std::string::npos if there is none @param[out] value the converted value @@ -10063,39 +10741,77 @@ template void convert_float_locale_aware(StringType& token, std::size_t decimal_point_position, FloatType& value) { const bool has_dot = decimal_point_position != std::string::npos; - char decimal_point = get_decimal_point(); + std::string decimal_point = get_decimal_point(); for (;;) { - const bool substitute = has_dot && decimal_point != '.'; - if (substitute) - { - token[decimal_point_position] = static_cast(decimal_point); - } - char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - strtof_by_type(value, token.data(), &endptr); - - if (substitute) + bool complete = false; + if (!has_dot || decimal_point.size() == 1) { - // the caller hands the token on (e.g. to the SAX interface) with '.' - token[decimal_point_position] = '.'; + const bool substitute = has_dot && decimal_point[0] != '.'; + if (substitute) + { + token[decimal_point_position] = static_cast(decimal_point[0]); + } + strtof_by_type(value, token.data(), &endptr); + if (substitute) + { + // the caller hands the token on (e.g. to the SAX interface) with '.' + token[decimal_point_position] = '.'; + } + complete = endptr == token.data() + token.size(); + } + else + { + std::string buffer(token.data(), token.size()); + buffer.replace(decimal_point_position, 1, decimal_point); + strtof_by_type(value, buffer.data(), &endptr); + complete = endptr == buffer.data() + buffer.size(); } - if (JSON_HEDLEY_LIKELY(endptr == token.data() + token.size())) + if (JSON_HEDLEY_LIKELY(complete)) { return; } // retry only if the locale changed; otherwise, this would loop forever - const char current_decimal_point = get_decimal_point(); + std::string current_decimal_point = get_decimal_point(); if (current_decimal_point == decimal_point) { return; } - decimal_point = current_decimal_point; + decimal_point = std::move(current_decimal_point); } } +/*! +@brief convert a validated float token like the lexer does + +For parsers of JSON text other than the lexer, which converts its own token +buffer in place. float, double, and long double where it is binary64 are +converted without allocation and independent of the locale; only other long +double formats that std::from_chars does not support need a copy of the token +for convert_float_locale_aware(). + +@param[in] first pointer to the first character of the token +@param[in] last pointer past the last character +@param[in] decimal_point_position index of the '.' in the token, or + std::string::npos if there is none +@param[in] mantissa_end index of the 'e'/'E', or the token length +@return the value, ±infinity if it overflows +*/ +template +FloatType convert_float(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end) +{ + FloatType value{}; + if (!convert_float_fast(first, last, decimal_point_position, mantissa_end, value)) + { + std::string token(first, last); + convert_float_locale_aware(token, decimal_point_position, value); + } + return value; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -10112,10 +10828,13 @@ NLOHMANN_JSON_NAMESPACE_END +#include // array #include // size_t -#include // uint64_t +#include // uint64_t, uint8_t #include // memcpy +// #include + // #include @@ -10174,18 +10893,12 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n) std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t word = 0; - std::memcpy(&word, data + i, sizeof(word)); - if (swar_string_special(word) != 0) + const std::uint64_t special = swar_string_special(read_eight_bytes(data + i)); + if (special != 0) { - // a special byte is in this word; locate it (endian-agnostic) - for (std::size_t j = 0; j < 8; ++j) - { - if (is_string_special(data[i + j])) - { - return i + j; - } - } + // the lowest flagged byte is the first special one: the borrows of + // the subtractions can only flag bytes above a true hit + return i + (static_cast(count_trailing_zeros(special)) / 8); } } for (; i < n; ++i) @@ -10219,8 +10932,7 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' (0x22) const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C) const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL (0x7F) @@ -10231,7 +10943,9 @@ inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_ | (v & high); // >= 0x80 if (stop != 0) { - break; + // the lowest flagged byte is the first one to stop at (see + // find_string_special()) + return i + (static_cast(count_trailing_zeros(stop)) / 8); } } for (; i < n; ++i) @@ -10358,12 +11072,18 @@ inline std::size_t scalar_string_bulk_run(const unsigned char* data, std::size_t { break; // end of buffer, or a quote/escape/control byte } - const std::size_t seq = validate_one_utf8(data + pos, n - pos); - if (seq == 0) + // a run of multi-byte sequences (e.g. CJK text) is validated sequence + // by sequence without searching for the next special byte in between + do { - break; // ill-formed or truncated: let the byte path diagnose it + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + return pos; // ill-formed or truncated: let the byte path diagnose it + } + pos += seq; } - pos += seq; + while (pos < n && data[pos] >= 0x80u); } return pos; } @@ -10378,8 +11098,7 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t std::size_t i = 0; for (; i + 8 <= n; i += 8) { - std::uint64_t v = 0; - std::memcpy(&v, data + i, sizeof(v)); + const std::uint64_t v = read_eight_bytes(data + i); const std::uint64_t q = v ^ 0x2222222222222222ull; const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; const std::uint64_t hit = ((q - ones) & ~q & high) @@ -10387,14 +11106,8 @@ inline std::size_t find_string_delimiter(const unsigned char* data, std::size_t | ((v - 0x2020202020202020ull) & ~v & high); if (hit != 0) { - for (std::size_t j = 0; j < 8; ++j) - { - const unsigned char c = data[i + j]; - if (c == '\"' || c == '\\' || c < 0x20u) - { - return i + j; - } - } + // the lowest flagged byte is the first delimiter (see find_string_special()) + return i + (static_cast(count_trailing_zeros(hit)) / 8); } } for (; i < n; ++i) @@ -10425,6 +11138,51 @@ inline std::size_t string_bulk_run(const unsigned char* data, std::size_t n) noe return scalar_string_bulk_run(data, n); } +// Decode the 4 hex digits at [data, data+4) - the digits following a `\u` +// escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte +// (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a +// hex digit ('0'..'9', 'A'..'F', 'a'..'f'). The caller must already have +// checked that 4 bytes are available; used by lexer::get_codepoint()'s +// contiguous fast path. On -1 it falls back to the byte-at-a-time loop, which +// stops at the first invalid digit, so the reported error and position are +// unaffected by this fast path. +inline int hex_codepoint(const unsigned char* data) noexcept +{ + static const std::array hex_digit_table = // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + { + { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 00..0F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 10..1F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 20..2F + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 30..3F ('0'..'9') + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 40..4F ('A'..'F') + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 50..5F + 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 60..6F ('a'..'f') + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 70..7F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 80..8F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 90..9F + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // A0..AF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // B0..BF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // C0..CF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // D0..DF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // E0..EF + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // F0..FF + } + }; + + const std::uint8_t d0 = hex_digit_table[data[0]]; + const std::uint8_t d1 = hex_digit_table[data[1]]; + const std::uint8_t d2 = hex_digit_table[data[2]]; + const std::uint8_t d3 = hex_digit_table[data[3]]; + // every valid digit is <= 0xF; the combined OR only exceeds it if at + // least one of the four bytes was not a hex digit (looked up as 0xFF) + if ((d0 | d1 | d2 | d3) > 0x0F) + { + return -1; + } + return (d0 << 12) | (d1 << 8) | (d2 << 4) | d3; +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -10596,6 +11354,15 @@ class lexer : public lexer_base static constexpr bool lazy_token_string = input_adapter_supports_seek(is_detected {}); + /// whether the read position is derived from the input adapter on demand + /// instead of being counted on every scanned character. Seekable adapters + /// know how much input they consumed and can revisit it, so + /// chars_read_total is the adapter's read count (minus a pending unget), + /// and line and column are recomputed from the consumed input only when + /// get_position() is called (errors). Counting per character kept two + /// increments and a newline check in the hot path of every get(). + static constexpr bool lazy_position = lazy_token_string; + /// whether a simulated unget can be passed on to the input adapter, which /// then leaves the character in the input; see /// input_adapter_supports_lookahead @@ -10605,7 +11372,9 @@ class lexer : public lexer_base /// whether string scanning may bulk-consume runs of ordinary characters /// directly from a contiguous input buffer (SWAR fast path). This requires /// the token to be reconstructible lazily (lazy_token_string), so bypassing - /// the per-character capture in get() cannot lose error diagnostics. + /// the per-character capture in get() cannot lose error diagnostics. It + /// also implies lazy_position, so the bulk paths need not advance the + /// position counters. static constexpr bool bulk_scan = lazy_token_string && input_adapter_supports_bulk_scan(is_detected {}); @@ -10631,6 +11400,41 @@ class lexer : public lexer_base // scan functions ///////////////////// + /// contiguous input: try to decode the 4 hex digits following `\\u` + /// directly from the input buffer via hex_codepoint(), instead of 4 calls + /// to get(). On success, advances the adapter exactly as those 4 get() + /// calls would (bulk_scan implies lazy_position, so there are no position + /// counters to advance) and leaves @a current holding the last of + /// the 4 digits, just as the last such get() would; the codepoint is + /// written to @a out. Makes no state change and returns false for a + /// pending unget, fewer than 4 remaining bytes, or any of the 4 bytes not + /// being a hex digit, so the caller falls back unchanged to the + /// per-character loop, which then reports the same diagnostic (stopping + /// at the first invalid digit) as before this optimization. + bool get_codepoint_bulk(std::true_type /*bulk*/, int& out) + { + if (next_unget || ia.bulk_remaining() < 4) + { + return false; + } + const char_type* const raw = ia.bulk_data(); + const int codepoint = hex_codepoint(reinterpret_cast(raw)); + if (codepoint < 0) + { + return false; + } + ia.bulk_skip(4); + current = char_traits::to_int_type(raw[3]); + out = codepoint; + return true; + } + + /// streaming input: no bulk fast path + bool get_codepoint_bulk(std::false_type /*bulk*/, int& /*out*/) const noexcept + { + return false; + } + /*! @brief get codepoint from 4 hex characters following `\\u` @@ -10650,6 +11454,14 @@ class lexer : public lexer_base { // this function only makes sense after reading `\u` JSON_ASSERT(current == 'u'); + + // contiguous input: decode all 4 hex digits directly from the buffer + int fast_codepoint = 0; + if (get_codepoint_bulk(std::integral_constant {}, fast_codepoint)) + { + return fast_codepoint; + } + int codepoint = 0; const auto factors = { 12u, 8u, 4u, 0u }; @@ -10741,10 +11553,6 @@ class lexer : public lexer_base } token_buffer.append(reinterpret_cast(data), pos); ia.bulk_skip(pos); - // the run contains no newline (all bytes < 0x20 are treated as special), - // so only the flat character counters advance - position.chars_read_total += pos; - position.chars_read_current_line += pos; } /// streaming input: no bulk fast path @@ -11468,9 +12276,11 @@ class lexer : public lexer_base token_type::parse_error otherwise @note The scanner is independent of the current locale: token_buffer - always holds `.`. Only the std::strtod fallback of convert_number() - depends on the locale, and it looks up the decimal point right - before converting (see detail::convert_float_locale_aware()). + always holds `.`. The conversion of float and double does not use + the locale either. Only the std::strtold fallback of + convert_number() for long double formats other than binary64 + depends on it, and it looks up the decimal point right before + converting (see detail::convert_float_locale_aware()). */ token_type scan_number() // lgtm [cpp/use-of-goto] `goto` is used in this function to implement the number-parsing state machine described above. By design, any finite input will eventually reach the "done" state or return token_type::parse_error. In each intermediate state, 1 byte of the input is appended to the token_buffer vector, and only the already initialized variables token_buffer, number_type, and error_message are manipulated. { @@ -11483,7 +12293,7 @@ class lexer : public lexer_base // offset just past the last mantissa byte in token_buffer (i.e. the // index of 'e'/'E', or the whole token when there is no exponent). - // convert_number() uses it to count significant digits; npos means + // convert_number() uses it to split the token; npos means // "not seen an exponent yet" and is resolved at scan_number_done std::size_t mantissa_end = std::string::npos; @@ -11813,8 +12623,8 @@ scan_number_done: @param[in] mantissa_end offset just past the last mantissa byte in token_buffer (the index of 'e'/'E', or token_buffer.size() when there is no exponent); - used to skip Clinger's fast path when it cannot - possibly succeed - see detail::mantissa_fits_clinger() + with decimal_point_position, it locates the parts + of a float token without scanning it again */ token_type convert_number(token_type number_type, std::size_t mantissa_end) { @@ -11868,10 +12678,11 @@ scan_number_done: } // this code is reached if we parse a floating-point number or if an - // integer conversion above overflowed. Prefer std::from_chars - // (Eisel-Lemire, locale-independent, correctly rounded) when available; - // otherwise the exact Clinger fast path (double only); otherwise the - // locale-aware strtof/strtod/strtold. + // integer conversion above overflowed. float and double (and long + // double where it is binary64) are converted by the library itself, + // correctly rounded and independent of the locale; other long double + // formats use std::from_chars when available, otherwise the + // locale-aware strtold. if (convert_float_fast(num_begin, num_end, decimal_point_position, mantissa_end, value_float)) { return token_type::value_float; @@ -11991,8 +12802,6 @@ scan_number_done: if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) { ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return integer_result; } // The value does not fit an integer, so this token converts as a @@ -12009,8 +12818,6 @@ scan_number_done: decimal_point_position = dot_index; ia.bulk_skip(len - 1); - position.chars_read_total += (len - 1); - position.chars_read_current_line += (len - 1); return convert_number(number_type, mantissa_end); } @@ -12061,7 +12868,7 @@ scan_number_done: #if JSON_DIAGNOSTIC_POSITIONS // the first character of the token has already been read, hence the -1 - token_start_position = position.chars_read_total - 1; + token_start_position = get_chars_read_total() - 1; #endif note_token_start(std::integral_constant {}); @@ -12114,6 +12921,15 @@ scan_number_done: /// per-character position counters (line-count-on-'\n' bookkeeping is /// handled afterwards, in track_after_read(), once `current` is known) void advance_position() noexcept + { + advance_position_impl(std::integral_constant {}); + } + + /// seekable adapter: the position is derived from the adapter on demand + void advance_position_impl(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: count the character + void advance_position_impl(std::false_type /*lazy*/) noexcept { ++position.chars_read_total; ++position.chars_read_current_line; @@ -12127,7 +12943,16 @@ scan_number_done: // seekable adapters reconstruct the token lazily on error (see // get_token_string), so the eager per-character copy is skipped capture_char(std::integral_constant {}); + track_newline(std::integral_constant {}); + return current; + } + /// seekable adapter: lines are counted on demand in get_position() + void track_newline(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: start a new line after reading a newline + void track_newline(std::false_type /*lazy*/) noexcept + { if (current == '\n') { ++position.lines_read; @@ -12136,8 +12961,6 @@ scan_number_done: chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } - - return current; } /*! @@ -12181,12 +13004,22 @@ scan_number_done: We implement unget by setting variable next_unget to true. The input is not changed - we just simulate ungetting by modifying chars_read_total, chars_read_current_line, and token_string. The next call to get() will - behave as if the unget character is read again. + behave as if the unget character is read again. With lazy_position, the + pending next_unget is all get_chars_read_total() needs. */ void unget() { next_unget = true; + unget_position(std::integral_constant {}); + uncapture_char(std::integral_constant {}); + } + /// seekable adapter: get_chars_read_total() accounts for next_unget + void unget_position(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: step the counters back over the ungotten character + void unget_position(std::false_type /*lazy*/) noexcept + { --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read @@ -12209,8 +13042,6 @@ scan_number_done: { --position.chars_read_current_line; } - - uncapture_char(std::integral_constant {}); } /// adapter without lookahead: nothing to do (see release_lookahead) @@ -12281,10 +13112,21 @@ scan_number_done: // diagnostics ///////////////////// - /// return position of last read token - constexpr position_t get_position() const noexcept + /*! + @brief return the number of characters read so far + + Unlike get_position(), this is O(1) for all adapters, so it may be called + for every value (JSON_DIAGNOSTIC_POSITIONS). + */ + std::size_t get_chars_read_total() const noexcept { - return position; + return get_chars_read_total_impl(std::integral_constant {}); + } + + /// return position of last read token + position_t get_position() const + { + return get_position_impl(std::integral_constant {}); } /*! @@ -12321,6 +13163,43 @@ scan_number_done: } #endif + private: + /// seekable adapter: every get() either read from the adapter (including + /// reads at EOF, which get() counts as characters as well) or replayed an + /// ungotten character, so the count is the adapter's read count minus a + /// still pending unget + std::size_t get_chars_read_total_impl(std::true_type /*lazy*/) const noexcept + { + return ia.get_read_count() - (next_unget ? 1u : 0u); + } + + /// streaming adapter: the counter is maintained by get() and unget() + std::size_t get_chars_read_total_impl(std::false_type /*lazy*/) const noexcept + { + return position.chars_read_total; + } + + /// seekable adapter: count the lines in the consumed input; O(input read + /// so far), but only called on errors + position_t get_position_impl(std::true_type /*lazy*/) const + { + position_t result; + result.chars_read_total = get_chars_read_total(); + // reads at EOF count as characters but are not part of the input + const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count()); + std::size_t line_start = 0; + result.lines_read = ia.count_consumed_lines(consumed, line_start); + result.chars_read_current_line = result.chars_read_total - line_start; + return result; + } + + /// streaming adapter: the position is maintained by get() and unget() + position_t get_position_impl(std::false_type /*lazy*/) const noexcept + { + return position; + } + + public: /// seekable adapter: rebuild the last read token from the input on demand const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const { @@ -12428,15 +13307,57 @@ scan_number_done: // into one that reloads the pointer from memory every character do { + if (current == '\n') + { + skip_indentation(std::integral_constant {}); + } get_ignoring_pending_unget(); } while (current_is_whitespace()); } + /*! + @brief contiguous input: skip the space indentation that follows a newline + 8 bytes at a time + + Pretty-printed JSON indents every line with a run of spaces, which the loop + in skip_whitespace() would otherwise read one character at a time. The + spaces are skipped 8 bytes at a time; the last 7 bytes of the input and any + other whitespace are left to that loop. Skipping bypasses get(), which is + fine because bulk_scan implies lazy_position: the position is derived from + the adapter, not counted per character. + */ + void skip_indentation(std::true_type /*bulk*/) + { + JSON_ASSERT(!next_unget); + const std::size_t remaining = ia.bulk_remaining(); + if (remaining < 8) + { + return; + } + const char* const data = reinterpret_cast(ia.bulk_data()); + std::size_t pos = 0; + for (; pos + 8 <= remaining; pos += 8) + { + // little-endian, so the first byte is the lowest + const std::uint64_t non_space = read_eight_bytes(data + pos) ^ 0x2020202020202020ull; + if (non_space != 0) + { + // skip the spaces before the first other byte of this word + pos += static_cast(count_trailing_zeros(non_space)) / 8; + break; + } + } + ia.bulk_skip(pos); + } + + /// streaming input: no bulk fast path + void skip_indentation(std::false_type /*bulk*/) const noexcept {} + token_type scan() { // initially, skip the BOM - if (position.chars_read_total == 0 && !skip_bom()) + if (get_chars_read_total() == 0 && !skip_bom()) { error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; return token_type::parse_error; @@ -12463,7 +13384,7 @@ scan_number_done: token_type scan_expecting(token_type expected_type) { JSON_ASSERT(expected_type == token_type::name_separator || expected_type == token_type::value_separator); - JSON_ASSERT(position.chars_read_total > 0); + JSON_ASSERT(get_chars_read_total() > 0); const char_int_type expected_char = static_cast((expected_type == token_type::name_separator) ? ':' : ','); skip_whitespace(); if (JSON_HEDLEY_LIKELY(current == expected_char)) @@ -13364,12 +14285,19 @@ Shared by json_sax_dom_parser and json_sax_dom_callback_parser. @param[in] ref the object or array that was just created @return whether parsing should continue (false after reporting out_of_range.408) */ +template +JSON_HEDLEY_NEVER_INLINE +bool container_too_large_error(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref) +{ + return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref)); +} + template bool check_container_size(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref) { if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref->max_size())) { - return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref)); + return container_too_large_error(sax, len, kind, ref); } return true; } @@ -13394,7 +14322,7 @@ struct diagnostic_positions { // Lexer has read the first character of the container, so // subtract 1 from the position to get the correct start position. - v.start_position = lexer->get_position() - 1; + v.start_position = lexer->get_chars_read_total() - 1; } } @@ -13408,7 +14336,7 @@ struct diagnostic_positions if (lexer) { // Lexer's position is past the closing brace or bracket, so set that as the end position. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); } } @@ -13424,7 +14352,7 @@ struct diagnostic_positions // Lexer has read past the current field value, so set the end position to the current position. // The start position will be set below based on the length of the string representation // of the value. - v.end_position = lexer->get_position(); + v.end_position = lexer->get_chars_read_total(); switch (v.type()) { @@ -13854,16 +14782,19 @@ class json_sax_dom_callback_parser if (keep && ref_stack.back()) { auto& obj = *ref_stack.back()->m_data.m_value.object; - const auto it = obj.find(val); - if (it != obj.end()) + // a single lookup: insert the placeholder, or find the existing member + const auto inserted = obj.emplace(val, discarded); + const auto it = inserted.first; + if (!inserted.second) { // this is a duplicate key (legal in JSON); remember its // current value so it can be restored later if the new // value is rejected by the callback, instead of being // erased together with the discarded placeholder duplicate_key_stash.emplace_back(&(it->second), it->second); + it->second = discarded; } - object_element = &(obj[val] = discarded); + object_element = &(it->second); } return true; @@ -14692,7 +15623,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current != char_traits::eof())) { - return last_byte_error(exception_id::unexpected_end_of_input, concat("expected end of input; last byte: 0x", get_token_string()), "value"); + return last_byte_error(exception_id::unexpected_end_of_input, "expected end of input", "value", ""); } } @@ -14823,7 +15754,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) { - static_cast(last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document")); + static_cast(bson_document_size_error(document_size, chars_read - document_start)); return repair_requested(); } return true; @@ -15147,7 +16078,7 @@ class binary_reader { // the end of the element is lost (see resync) skip_requested = true; - return last_byte_error(exception_id::unexpected_byte, concat("string length must be at least 1, is ", std::to_string(len)), "string"); + return bson_length_error("string length must be at least 1, is ", len, "string"); } if (JSON_HEDLEY_UNLIKELY(!get_string(len - static_cast(1), result))) @@ -15182,7 +16113,7 @@ class binary_reader { // the end of the element is lost (see resync) skip_requested = true; - return last_byte_error(exception_id::unexpected_byte, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"); + return bson_length_error("byte array length cannot be negative, is ", len, "binary"); } // All BSON binary values have a subtype @@ -15272,9 +16203,7 @@ class binary_reader default: // anything else is not supported (yet) { - const std::string cr_str = hex_byte(static_cast(element_type)); - static_cast(report_error(element_type_parse_position, cr_str, - parse_error::create(exception_id::bson_unsupported_type, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr))); + static_cast(bson_unsupported_type_error(element_type_parse_position, element_type)); return skip_unsupported_bson_element(element_type); } } @@ -15728,6 +16657,7 @@ class binary_reader @param[in] context parsing context for the error message @return false, so that the caller stops reading */ + JSON_HEDLEY_NEVER_INLINE bool cbor_indefinite_string_error(const char* type_name, const char* context) { return last_byte_error(exception_id::invalid_string_or_size, concat("indefinite-length ", type_name, " is not allowed inside indefinite-length ", type_name, "; last byte: 0x", get_token_string()), context); @@ -15807,7 +16737,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x60-0x7B)", inside_indefinite ? "" : " or indefinite string type (0x7F)", "; last byte: 0x", get_token_string()), "string"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x60-0x7B)", "string", inside_indefinite ? "" : " or indefinite string type (0x7F)"); } } } @@ -15941,7 +16871,7 @@ class binary_reader // a key that begins an item is skipped with its value when recovering // (see resync); a break stop code or a reserved byte begins none skip_requested = is_cbor_item_head(static_cast(current)); - return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key"); + return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found); } /*! @@ -16022,7 +16952,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x40-0x5B)", inside_indefinite ? "" : " or indefinite binary array type (0x5F)", "; last byte: 0x", get_token_string()), "binary"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x40-0x5B)", "binary", inside_indefinite ? "" : " or indefinite binary array type (0x5F)"); } } } @@ -16168,8 +17098,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message(concat("excessive ", context, " size"), "size"), nullptr)); + return container_size_error(context); } result = conditional_static_cast(len); return true; @@ -16857,7 +17786,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", get_token_string()), "string"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0xA0-0xBF, 0xD9-0xDB)", "string", ""); } } } @@ -16952,7 +17881,7 @@ class binary_reader // the key is skipped with its value when recovering (see resync) skip_requested = true; - return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key"); + return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found); } /*! @@ -17597,8 +18526,7 @@ class binary_reader } if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(number))) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("integer value overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char return true; @@ -17695,8 +18623,7 @@ class binary_reader } if (!value_in_range_of(number)) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("integer value overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } result = detail::conditional_static_cast(number); return true; @@ -17777,7 +18704,7 @@ class binary_reader // or SIZE_MAX. if (JSON_HEDLEY_UNLIKELY(result > (std::numeric_limits::max)() / i)) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size"); } result *= i; // the pre-check above already rules out result becoming 0 @@ -17786,7 +18713,7 @@ class binary_reader // unknown-size container (see get_ubjson_size_type()) if (result == npos) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size"); } if (JSON_HEDLEY_UNLIKELY(!emit_unsigned(i))) { @@ -17837,7 +18764,7 @@ class binary_reader if (input_format == input_format_t::bjdata && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) { - return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type"); + return optimized_type_error(); } if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) @@ -17852,7 +18779,7 @@ class binary_reader { return false; } - return last_byte_error(exception_id::unexpected_byte, concat("expected '#' after type information; last byte: 0x", get_token_string()), "size"); + return last_byte_error(exception_id::unexpected_byte, "expected '#' after type information", "size", ""); } const bool is_error = get_ubjson_size_value(result.first, is_ndarray, 0, result.second); @@ -18003,7 +18930,7 @@ class binary_reader } if (JSON_HEDLEY_UNLIKELY(current > 127)) { - static_cast(last_byte_error(exception_id::invalid_string_or_size, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", get_token_string()), "char")); + static_cast(last_byte_error(exception_id::invalid_string_or_size, "byte after 'C' must be in range 0x00..0x7F", "char", "")); if (!repair_requested()) { return false; @@ -18097,8 +19024,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') && size_and_type.first > max_valueless_container_size)) { - return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("excessive array size", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive array size", "size"); } if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) @@ -19009,7 +19935,8 @@ class binary_reader { // in case of failure, advance position by 1 to report the failing location ++chars_read; - return report_error(chars_read, "", parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + static_cast(report_eof(context)); + return false; } return true; } @@ -19161,8 +20088,7 @@ class binary_reader const auto result = static_cast(number); if (JSON_HEDLEY_UNLIKELY(std::isfinite(number) && !std::isfinite(result))) { - static_cast(report_error(chars_read, get_token_string(), - out_of_range::create(exception_id::number_overflow, exception_message("number overflow", "value"), nullptr))); + static_cast(out_of_range_error(exception_id::number_overflow, "number overflow", "value")); // when recovering, the number becomes infinity, as in JSON text if (!repair_requested()) { @@ -19682,22 +20608,82 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(current == char_traits::eof())) { - return report_error(chars_read, "", - parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + return report_eof(context); } return true; } + /*! + @brief reports an unexpected end of input (out of line to keep the exception + construction out of the hot reading functions) + @param[in] context further context information (for diagnostics) + @return false, so that the caller stops reading (see report_error) + */ + JSON_HEDLEY_NON_NULL(2) + JSON_HEDLEY_NEVER_INLINE + bool report_eof(const char* context) + { + return report_error(chars_read, "", + parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + } + + /*! + @brief reports an out_of_range error at the last read byte (out of line) + */ + JSON_HEDLEY_NEVER_INLINE + bool out_of_range_error(const exception_id id_, const char* detail, const char* context) + { + return report_error(chars_read, get_token_string(), out_of_range::create(id_, exception_message(detail, context), nullptr)); + } + + JSON_HEDLEY_NEVER_INLINE + bool container_size_error(const char* what) + { + return report_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, + exception_message(concat("excessive ", what, " size"), "size"), nullptr)); + } + + JSON_HEDLEY_NEVER_INLINE + bool optimized_type_error() + { + return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type"); + } + + template + JSON_HEDLEY_NEVER_INLINE + bool bson_length_error(const char* prefix, const NumberType len, const char* context) + { + return last_byte_error(exception_id::unexpected_byte, concat(prefix, std::to_string(len)), context); + } + + JSON_HEDLEY_NEVER_INLINE + bool bson_document_size_error(const std::int32_t document_size, const std::size_t bytes_read) + { + return last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(bytes_read), ")"), "document"); + } + + JSON_HEDLEY_NEVER_INLINE + bool bson_unsupported_type_error(const std::size_t position, const char element_type) + { + const std::string cr_str = hex_byte(static_cast(element_type)); + return report_error(position, cr_str, + parse_error::create(exception_id::bson_unsupported_type, position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + } + /*! @brief reports a parse error at the last read byte @param[in] id_ the id of the parse_error exception @param[in] detail a detailed error message @param[in] context further context information + @param[in] tail optional text appended to the detail, followed by "; last byte: 0x" @return false, so that the caller stops reading (see report_error) */ - bool last_byte_error(const exception_id id_, const std::string& detail, const std::string& context) + JSON_HEDLEY_NEVER_INLINE + bool last_byte_error(const exception_id id_, const std::string& detail, const char* context, const char* tail = nullptr) { - return report_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, exception_message(detail, context), nullptr)); + // with a tail, the message reads "; last byte: 0x" + return report_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, + exception_message(tail == nullptr ? detail : concat(detail, tail, "; last byte: 0x", get_token_string()), context), nullptr)); } /*! @@ -19706,7 +20692,8 @@ class binary_reader @param[in] context further context information @return false, so that the caller stops reading (see report_error) */ - bool unexpected_byte(const char* detail, const std::string& context) + JSON_HEDLEY_NEVER_INLINE + bool unexpected_byte(const char* detail, const char* context) { return last_byte_error(exception_id::unexpected_byte, concat(detail, ": 0x", get_token_string()), context); } @@ -19717,6 +20704,7 @@ class binary_reader @param[in] context further context information @return false, so that the caller stops reading (see report_error) */ + JSON_HEDLEY_NEVER_INLINE bool length_type_error(const char* position, const char* context) { const char* types = input_format == input_format_t::bjdata ? "U, i, u, I, m, l, M, L" : "U, i, I, l, L"; @@ -20355,7 +21343,7 @@ class parser } case token_type::end_of_input: { - if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1)) + if (JSON_HEDLEY_UNLIKELY(m_lexer.get_chars_read_total() == 1)) { // there is nothing to recover static_cast(report_error(sax, parse_error::create(exception_id::syntax_error, m_lexer.get_position(), @@ -21120,8 +22108,10 @@ class parser { // MSVC 2015 reports C4100 (unreferenced parameter) when SAX::parse_error is static static_cast(sax); - return sax.parse_error(m_lexer.get_position(), m_lexer.get_token_string(), - parse_error::create(exception_id::syntax_error, m_lexer.get_position(), message, nullptr)); + // computing the position is O(input read so far) for seekable input + const position_t position = m_lexer.get_position(); + return sax.parse_error(position, m_lexer.get_token_string(), + parse_error::create(exception_id::syntax_error, position, message, nullptr)); } std::string exception_message(const token_type expected, const std::string& context) @@ -21637,7 +22627,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci } /// @throw invalid_iterator.214 always - JSON_HEDLEY_NO_RETURN void throw_cannot_get_value() const + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_cannot_get_value() const { JSON_THROW(invalid_iterator::create(exception_id::iterator_value_unavailable, "cannot get value", m_object)); } @@ -21651,40 +22641,25 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); - return m_it.object_iterator->second; - } - - case value_t::array: - { - JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); - return *m_it.array_iterator; - } - - case value_t::null: - throw_cannot_get_value(); - - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) - { - return *m_object; - } - - throw_cannot_get_value(); - } + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); + return *m_it.array_iterator; } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); + return m_it.object_iterator->second; + } + + // neither array nor object: a primitive value (null has no value to point to) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type != value_t::null && m_it.primitive_iterator.is_begin())) + { + return *m_object; + } + + throw_cannot_get_value(); } /*! @@ -21695,38 +22670,25 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); - return &(m_it.object_iterator->second); - } - - case value_t::array: - { - JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); - return &*m_it.array_iterator; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) - { - return m_object; - } - - throw_cannot_get_value(); - } + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); + return &*m_it.array_iterator; } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); + return &(m_it.object_iterator->second); + } + + // neither array nor object: a primitive value + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return m_object; + } + + throw_cannot_get_value(); } /*! @@ -21748,35 +22710,20 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - std::advance(m_it.object_iterator, 1); - break; - } - - case value_t::array: - { - std::advance(m_it.array_iterator, 1); - break; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - ++m_it.primitive_iterator; - break; - } + ++m_it.array_iterator; + return *this; } + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + std::advance(m_it.object_iterator, 1); + return *this; + } + + // neither array nor object: a primitive value + ++m_it.primitive_iterator; return *this; } @@ -21799,35 +22746,20 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - std::advance(m_it.object_iterator, -1); - break; - } - - case value_t::array: - { - std::advance(m_it.array_iterator, -1); - break; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - --m_it.primitive_iterator; - break; - } + --m_it.array_iterator; + return *this; } + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + std::advance(m_it.object_iterator, -1); + return *this; + } + + // neither array nor object: a primitive value + --m_it.primitive_iterator; return *this; } @@ -21841,7 +22773,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci // if objects are not the same, the comparison is undefined if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) { - JSON_THROW(invalid_iterator::create(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object)); + throw_literal(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object); } // value-initialized forward iterators can be compared, and must compare equal to other value-initialized iterators of the same type #4493 @@ -21850,25 +22782,18 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci return true; } - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - return (m_it.object_iterator == other.m_it.object_iterator); - - case value_t::array: - return (m_it.array_iterator == other.m_it.array_iterator); - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - return (m_it.primitive_iterator == other.m_it.primitive_iterator); + return (m_it.array_iterator == other.m_it.array_iterator); } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + // neither array nor object: a primitive value + return (m_it.primitive_iterator == other.m_it.primitive_iterator); } /*! @@ -21890,7 +22815,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci // if objects are not the same, the comparison is undefined if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) { - JSON_THROW(invalid_iterator::create(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object)); + throw_literal(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object); } // value-initialized forward iterators can be compared, and must compare equal to other value-initialized iterators of the same type #4493 @@ -21903,7 +22828,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_order_on_object, "cannot compare order of object iterators", m_object)); + throw_literal(exception_id::iterator_order_on_object, "cannot compare order of object iterators", m_object); case value_t::array: return (m_it.array_iterator < other.m_it.array_iterator); @@ -21959,7 +22884,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object)); + throw_literal(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object); case value_t::array: { @@ -22038,7 +22963,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object)); + throw_literal(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object); case value_t::array: return m_it.array_iterator - other.m_it.array_iterator; @@ -22067,7 +22992,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_subscript_on_object, "cannot use operator[] for object iterators", m_object)); + throw_literal(exception_id::iterator_subscript_on_object, "cannot use operator[] for object iterators", m_object); case value_t::array: return *std::next(m_it.array_iterator, n); @@ -22107,7 +23032,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci return m_it.object_iterator->first; } - JSON_THROW(invalid_iterator::create(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object)); + throw_literal(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object); } /*! @@ -22339,8 +23264,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // ostream #endif // JSON_NO_IO #include // max -#include // map -#include // accumulate #include // string #include // move #include // vector @@ -22349,6 +23272,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -22398,12 +23323,21 @@ class json_pointer /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ string_t to_string() const { - return std::accumulate(reference_tokens.begin(), reference_tokens.end(), - string_t{}, - [](const string_t& a, const string_t& b) + // one buffer: every token adds its escaped form (at least its length) and a slash + std::size_t length = 0; + for (const auto& reference_token : reference_tokens) { - return detail::concat(a, '/', detail::escape(b)); - }); + length += reference_token.size() + 1; + } + + string_t result; + result.reserve(length); + for (const auto& reference_token : reference_tokens) + { + result.append("/", 1); + detail::escape_into(result, reference_token); + } + return result; } /// @brief return a string representation of the JSON pointer @@ -22458,21 +23392,29 @@ class json_pointer friend json_pointer operator/(const json_pointer& lhs, const json_pointer& rhs) { - return json_pointer(lhs) /= rhs; + json_pointer result; + result.reference_tokens.reserve(lhs.reference_tokens.size() + rhs.reference_tokens.size()); + result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); + result.reference_tokens.insert(result.reference_tokens.end(), rhs.reference_tokens.begin(), rhs.reference_tokens.end()); + return result; } /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ friend json_pointer operator/(const json_pointer& lhs, string_t token) // NOLINT(performance-unnecessary-value-param) { - return json_pointer(lhs) /= std::move(token); + json_pointer result; + result.reference_tokens.reserve(lhs.reference_tokens.size() + 1); + result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); + result.reference_tokens.push_back(std::move(token)); + return result; } /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx) { - return json_pointer(lhs) /= array_idx; + return lhs / detail::to_string(array_idx); } /// @brief returns the parent of this JSON pointer @@ -22484,8 +23426,8 @@ class json_pointer return *this; } - json_pointer res = *this; - res.pop_back(); + json_pointer res; + res.reference_tokens = std::vector(reference_tokens.begin(), reference_tokens.end() - 1); return res; } @@ -22574,14 +23516,14 @@ class json_pointer private: /// @throw out_of_range.405 always - JSON_HEDLEY_NO_RETURN static void throw_no_parent() + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_no_parent() { JSON_THROW(detail::out_of_range::create(detail::exception_id::patch_on_root, "JSON pointer has no parent", nullptr)); } /// @throw out_of_range.404 always template - JSON_HEDLEY_NO_RETURN static void throw_unresolved(const string_t& reference_token, BasicJsonContext context) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_unresolved(const string_t& reference_token, BasicJsonContext context) { static_cast(reference_token); // unused when JSON_NOEXCEPTION is defined static_cast(context); @@ -22634,6 +23576,28 @@ class json_pointer return array_index_status::not_a_number; } + // fast path: up to 18 decimal digits cannot overflow unsigned long long or + // size_type (9 digits for a 32-bit size_type), so strtoull and errno are not needed + constexpr std::size_t max_fast_digits = sizeof(size_type) >= 8 ? 18 : 9; + if (!s.empty() && s.size() <= max_fast_digits) + { + size_type acc = 0; + for (std::size_t i = 0; i < s.size(); ++i) + { + const char c = s[i]; + if (JSON_HEDLEY_UNLIKELY(c < '0' || c > '9')) + { + break; // not a plain number: let strtoull decide + } + acc = static_cast(acc * 10 + static_cast(c - '0')); + if (JSON_HEDLEY_UNLIKELY(i == s.size() - 1)) + { + idx = acc; + return array_index_status::ok; + } + } + } + const char* p = s.data(); char* p_end = nullptr; // NOLINT(misc-const-correctness) errno = 0; // strtoull doesn't reset errno @@ -22690,6 +23654,83 @@ class json_pointer return idx; } + /*! + @brief compare with a JSON pointer string without splitting it into tokens + + @throw parse_error.107 if the string is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ + bool equals_string(const string_t& reference_string) const + { + if (reference_string.empty()) + { + return reference_tokens.empty(); + } + + if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/')) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr)); + } + + // the whole string is validated, whether or not it matches + const std::size_t size = reference_string.size(); + for (std::size_t pos = reference_string.find_first_of('~'); + pos != string_t::npos; + pos = reference_string.find_first_of('~', pos + 1)) + { + if (JSON_HEDLEY_UNLIKELY(pos == size - 1 || (reference_string[pos + 1] != '0' && reference_string[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + } + } + + // compare token by token, unescaping the string on the fly + std::size_t pos = 0; // the slash in front of the next token + for (const auto& reference_token : reference_tokens) + { + if (pos >= size) + { + return false; + } + + std::size_t i = pos + 1; + std::size_t j = 0; + const std::size_t token_size = reference_token.size(); + while (i < size && reference_string[i] != '/') + { + char c = reference_string[i++]; + if (c == '~') + { + c = (reference_string[i++] == '0') ? '~' : '/'; + } + if (j == token_size || reference_token[j] != c) + { + return false; + } + ++j; + } + if (j != token_size) + { + return false; + } + pos = i; + } + return pos == size; + } + + /// @brief object member access with a key that is only copied if it is inserted + template + static BasicJsonType& object_member(BasicJsonType& object, const string_t& key) + { + auto& members = *object.m_data.m_value.object; + const auto it = members.find(key); + if (it != members.end()) + { + return it->second; + } + return object.operator[](key); + } + JSON_PRIVATE_UNLESS_TESTED: json_pointer top() const { @@ -22713,84 +23754,141 @@ class json_pointer the result independent of the order in which the flattened object is iterated, which is unspecified for some object types. - The prefixes form a tree and are numbered, so each of them is stored only - once (as a node) rather than as a copy of all of its reference tokens. + The prefixes form a tree and are numbered (number 0 is the empty prefix), + so each of them is stored only once (as a node) rather than as a copy of + all of its reference tokens. The tree does not own the reference tokens: + it refers to the tokens of the parsed keys, which must outlive it. */ struct prefix_tree { - // children[id] maps a reference token to the number of the prefix - // extended by that token; number 0 is the empty prefix - std::vector> children; + // open-addressing hash table from (parent, last reference token) to + // the number of the node; empty slots have token == nullptr + struct slot + { + std::size_t hash; + std::size_t parent; + const string_t* token; + std::size_t id; + + slot() noexcept + : hash(0), parent(0), token(nullptr), id(0) + {} + + slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept + : hash(h), parent(p), token(t), id(i) + {} + }; + + std::vector slots; + std::size_t used = 0; + + static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept + { + std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL); + for (std::size_t i = 0; i < token.size(); ++i) + { + h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL); + } + return h ^ (h >> 29u); + } + + void grow() + { + std::vector old(slots.size() * 2); + old.swap(slots); + const std::size_t mask = slots.size() - 1; + for (const auto& s : old) + { + if (s.token != nullptr) + { + std::size_t i = s.hash & mask; + while (slots[i].token != nullptr) + { + i = (i + 1) & mask; + } + slots[i] = s; + } + } + } + + // parent[id] is the number of the prefix without its last token + std::vector parent; // is_array[id] is true iff some flattened key has the reference token // 0 directly below the prefix with number id std::vector is_array; // start with the empty prefix only prefix_tree() - : children(1) + : slots(16) + , parent(1, 0) , is_array(1, false) {} // return the number of the prefix with number id extended by // reference_token, adding it if it is new - std::size_t add_child(std::size_t id, string_t&& reference_token) + std::size_t add_child(std::size_t id, const string_t& reference_token) { + const std::size_t hash = hash_of(id, reference_token); + const std::size_t mask = slots.size() - 1; + std::size_t i = hash & mask; + while (slots[i].token != nullptr) + { + if (slots[i].hash == hash && slots[i].parent == id && *slots[i].token == reference_token) + { + return slots[i].id; + } + i = (i + 1) & mask; + } + + // new prefix + const std::size_t child = parent.size(); + slots[i] = slot(hash, id, &reference_token, child); if (reference_token == "0") { is_array[id] = true; } + parent.push_back(id); + is_array.push_back(false); - // read the number before the emplace_back below, which may - // reallocate children and invalidate the iterator - const std::size_t next = children.size(); - const auto inserted = children[id].emplace(std::move(reference_token), next); - const std::size_t child = inserted.first->second; - if (inserted.second) + // keep the load factor below 1/2 + if (++used * 2 > slots.size()) { - children.emplace_back(); - is_array.push_back(false); + grow(); } return child; } - - // return the number of the prefix with number id extended by - // reference_token, which must have been added before - std::size_t find_child(std::size_t id, const string_t& reference_token) const - { - const auto it = children[id].find(reference_token); - JSON_ASSERT(it != children[id].end()); - return it->second; - } }; /*! @brief create and return a reference to the pointed to value - Complexity: Linear in the number of reference tokens (times the logarithm - of the number of siblings for the prefix lookup). + @param[in] j the value to create the pointed to value in + @param[in] tokens the reference tokens of the pointer + @param[in] prefixes prefixes[i] is the number in @a tree of the prefix + consisting of the first i reference tokens + + Complexity: Linear in the number of reference tokens. @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ template - BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const + static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector& tokens, + const std::vector& prefixes, const prefix_tree& tree) { auto* result = &j; - // the number of the prefix consumed so far; used to look up whether - // the value to be created below is an array or an object - std::size_t id = 0; - // in case no reference tokens exist, return a reference to the JSON value // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) + for (std::size_t i = 0; i < tokens.size(); ++i) { + const auto& reference_token = tokens[i]; switch (result->type()) { case detail::value_t::null: { - if (tree.is_array[id]) + if (tree.is_array[prefixes[i]]) { // some reference token below this position is 0, so the // value is an array @@ -22807,7 +23905,7 @@ class json_pointer case detail::value_t::object: { // create an entry in the object - result = &result->operator[](reference_token); + result = &object_member(*result, reference_token); break; } @@ -22834,8 +23932,6 @@ class json_pointer default: JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j)); } - - id = tree.find_child(id, reference_token); } return *result; @@ -22887,7 +23983,7 @@ class json_pointer case detail::value_t::object: { // use unchecked object access - ptr = &ptr->operator[](reference_token); + ptr = &object_member(*ptr, reference_token); break; } @@ -22937,8 +24033,13 @@ class json_pointer { case detail::value_t::object: { - // note: at performs range check - ptr = &ptr->at(reference_token); + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) + { + // at() throws out_of_range.403 with the usual message + ptr->at(reference_token); + } + ptr = &it->second; break; } @@ -22959,7 +24060,7 @@ class json_pointer JSON_THROW(detail::out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat( "array index ", std::to_string(idx), " is out of range"), ptr)); } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -23007,7 +24108,9 @@ class json_pointer { // use unchecked object access; the const operator[] // asserts that the key exists - ptr = &ptr->operator[](reference_token); + const auto it = ptr->m_data.m_value.object->find(reference_token); + JSON_ASSERT(it != ptr->m_data.m_value.object->end()); + ptr = &it->second; break; } @@ -23021,7 +24124,9 @@ class json_pointer // use unchecked array access; the const operator[] // asserts that the index exists - ptr = &ptr->operator[](array_index(reference_token)); + const auto idx = array_index(reference_token); + JSON_ASSERT(idx < ptr->m_data.m_value.array->size()); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -23062,12 +24167,12 @@ class json_pointer { case detail::value_t::object: { - const auto it = ptr->find(reference_token); - if (JSON_HEDLEY_UNLIKELY(it == ptr->end())) + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) { return nullptr; } - ptr = &*it; + ptr = &it->second; break; } @@ -23104,7 +24209,7 @@ class json_pointer { return nullptr; } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -23137,13 +24242,14 @@ class json_pointer { case detail::value_t::object: { - if (!ptr->contains(reference_token)) + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (it == ptr->m_data.m_value.object->end()) { // we did not find the key in the object return false; } - ptr = &ptr->operator[](reference_token); + ptr = &it->second; break; } @@ -23165,13 +24271,13 @@ class json_pointer return false; } - if (idx >= ptr->size()) + if (idx >= ptr->m_data.m_value.array->size()) { // index out of range return false; } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -23221,6 +24327,15 @@ class json_pointer JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr)); } + // every slash starts a reference token + const std::size_t size = reference_string.size(); + std::size_t token_count = 0; + for (std::size_t i = 0; i < size; ++i) + { + token_count += (reference_string[i] == '/') ? 1 : 0; + } + result.reserve(token_count); + // extract the reference tokens: // - slash: position of the last read slash (or end of string) // - start: position after the previous slash @@ -23242,31 +24357,52 @@ class json_pointer const auto count = (slash == string_t::npos ? reference_string.size() : slash) - start; auto reference_token = string_t(reference_string.data() + start, count); - // check reference tokens are properly escaped - for (std::size_t pos = reference_token.find_first_of('~'); - pos != string_t::npos; - pos = reference_token.find_first_of('~', pos + 1)) + // check reference tokens are properly escaped; a token without + // a '~' (the common case) needs neither check nor unescaping + std::size_t pos = reference_token.find_first_of('~'); + if (pos != string_t::npos) { - JSON_ASSERT(reference_token[pos] == '~'); - - // ~ must be followed by 0 or 1 - if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || - (reference_token[pos + 1] != '0' && - reference_token[pos + 1] != '1'))) + for (; pos != string_t::npos; pos = reference_token.find_first_of('~', pos + 1)) { - JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + JSON_ASSERT(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || + (reference_token[pos + 1] != '0' && + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + } } + + detail::unescape(reference_token); } // finally, store the reference token - detail::unescape(reference_token); - result.push_back(reference_token); + result.push_back(std::move(reference_token)); } return result; } private: + // Add the entry (key, v) to the object `obj`. Flattened paths are pairwise + // distinct, so objects that can append a key known to be absent + // (ordered_map) skip the lookup, which is linear and would make flatten + // quadratic; all other object types use operator[]. + template + static auto add_flattened(ObjectType& obj, const string_t& key, ValueType&& v, detail::priority_tag<1> /*unused*/) + -> decltype(obj.emplace_new_key(key, std::forward(v)), void()) + { + obj.emplace_new_key(key, std::forward(v)); + } + + template + static void add_flattened(ObjectType& obj, const string_t& key, ValueType&& v, detail::priority_tag<0> /*unused*/) + { + obj[key] = std::forward(v); + } + /*! @param[in] reference_string the reference string to the current value @param[in] value the value to consider @@ -23311,7 +24447,8 @@ class json_pointer // flatten `v`, whose path is `path`: primitives and empty containers // are added to the result right away; other containers get a frame - const auto enter = [&stack, &path, &result](const BasicJsonType & v) + auto& object = *result.m_data.m_value.object; + const auto enter = [&stack, &path, &object](const BasicJsonType & v) { switch (v.type()) { @@ -23320,7 +24457,7 @@ class json_pointer if (v.m_data.m_value.array->empty()) { // flatten empty array as null - result[path] = nullptr; + add_flattened(object, path, nullptr, detail::priority_tag<1> {}); } else { @@ -23334,7 +24471,7 @@ class json_pointer if (v.m_data.m_value.object->empty()) { // flatten empty object as null - result[path] = nullptr; + add_flattened(object, path, nullptr, detail::priority_tag<1> {}); } else { @@ -23354,7 +24491,7 @@ class json_pointer default: { // add a primitive value with its reference string - result[path] = v; + add_flattened(object, path, v, detail::priority_tag<1> {}); return; } } @@ -23400,6 +24537,10 @@ class json_pointer enter(it->second); } } + + // the entries were added without going through basic_json, so the + // parent pointers (JSON_DIAGNOSTICS) have to be set now + result.set_parents(); } /*! @@ -23423,21 +24564,53 @@ class json_pointer BasicJsonType result; + // parse every key once; the reference tokens are kept for the second + // pass (the vectors are never resized, so the tree may refer to them) + std::vector> keys; + keys.reserve(value.m_data.m_value.object->size()); + for (const auto& element : *value.m_data.m_value.object) + { + json_pointer ptr(element.first); + keys.push_back(std::move(ptr.reference_tokens)); + } + // collect the pointer prefixes that have a reference token 0 among // their children; the values below them are arrays, all others are // objects (see prefix_tree) prefix_tree tree; - for (const auto& element : *value.m_data.m_value.object) + // the number of the prefix with i + 1 tokens of the previous key; keys + // are often sorted, so most of the prefix is the same as for the + // previous key and need not be looked up again + std::vector path; + const std::vector* previous = nullptr; + // the number of the complete prefix (i.e., the key) of each key + std::vector leaves; + leaves.reserve(keys.size()); + for (const auto& tokens : keys) { - json_pointer ptr(element.first); - std::size_t id = 0; - for (auto& reference_token : ptr.reference_tokens) + std::size_t common = 0; + if (previous != nullptr) { - id = tree.add_child(id, std::move(reference_token)); + const std::size_t limit = (std::min)(tokens.size(), previous->size()); + while (common < limit && tokens[common] == (*previous)[common]) + { + ++common; + } } + path.resize(common); + std::size_t id = common == 0 ? 0 : path[common - 1]; + for (std::size_t i = common; i < tokens.size(); ++i) + { + id = tree.add_child(id, tokens[i]); + path.push_back(id); + } + leaves.push_back(id); + previous = &tokens; } // iterate the JSON object values + std::vector prefixes; + std::size_t key_index = 0; for (const auto& element : *value.m_data.m_value.object) { if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) @@ -23445,11 +24618,21 @@ class json_pointer JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second)); } + const auto& tokens = keys[key_index]; + // number the prefixes of this key from its last one to the first + prefixes.resize(tokens.size()); + for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i) + { + id = tree.parent[id]; + prefixes[i - 1] = id; + } + ++key_index; + // Assign the value to the reference pointed to by JSON pointer. Note // that if the JSON pointer is "" (i.e., points to the whole value), // function get_and_create returns a reference to the result itself. // An assignment will then create a primitive value. - json_pointer(element.first).get_and_create(result, tree) = element.second; + get_and_create(result, tokens, prefixes, tree) = element.second; } return result; @@ -23488,7 +24671,7 @@ class json_pointer = delete; #else { - return *this == json_pointer(rhs); + return equals_string(rhs); } #endif @@ -23572,7 +24755,7 @@ inline bool operator==(const json_pointer& lhs, = delete; #else { - return lhs == json_pointer(rhs); + return lhs.equals_string(rhs); } #endif @@ -23585,7 +24768,7 @@ inline bool operator==(const StringType& lhs, = delete; #else { - return json_pointer(lhs) == rhs; + return rhs.equals_string(lhs); } #endif @@ -23722,9 +24905,9 @@ class json_ref } // namespace detail NLOHMANN_JSON_NAMESPACE_END - +// IWYU pragma: export // #include - +// IWYU pragma: export // #include // #include @@ -24266,7 +25449,7 @@ class binary_writer case value_t::number_unsigned: { - write_cbor_head(0x00, j.m_data.m_value.number_unsigned); + write_cbor_head(0x00, conditional_static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -25646,7 +26829,7 @@ class binary_writer return calc_bson_integer_size(j.m_data.m_value.number_integer); case value_t::number_unsigned: - return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); + return calc_bson_unsigned_size(conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return calc_bson_string_size(*j.m_data.m_value.string, j); @@ -25688,7 +26871,7 @@ class binary_writer return write_bson_integer(name, j.m_data.m_value.number_integer); case value_t::number_unsigned: - return write_bson_unsigned(name, j.m_data.m_value.number_unsigned); + return write_bson_unsigned(name, conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return write_bson_string(name, *j.m_data.m_value.string); @@ -27358,6 +28541,7 @@ NLOHMANN_JSON_NAMESPACE_END // |_____|_____|_____|_|___| https://github.com/nlohmann/json // // SPDX-FileCopyrightText: 2009 Florian Loitsch +// SPDX-FileCopyrightText: 2025 Victor Zverovich // SPDX-FileCopyrightText: 2013-2026 Niels Lohmann // SPDX-License-Identifier: MIT @@ -27365,11 +28549,259 @@ NLOHMANN_JSON_NAMESPACE_END #include // array #include // signbit, isfinite +#include // size_t #include // intN_t, uintN_t #include // memcpy, memmove #include // numeric_limits #include // conditional +#ifdef _MSC_VER + #include // _byteswap_uint64 +#endif + +// SSE2 (every x86-64 CPU) and NEON (every 64-bit Arm CPU) convert the 16 +// digits of a double at once +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) + #include + #define JSON_DTOA_SSE2 1 + #define JSON_DTOA_NEON 0 +#elif (defined(__aarch64__) || defined(_M_ARM64)) && !defined(_M_ARM64EC) && !defined(__ARM_BIG_ENDIAN) + #include + #define JSON_DTOA_SSE2 0 + #define JSON_DTOA_NEON 1 +#else + #define JSON_DTOA_SSE2 0 + #define JSON_DTOA_NEON 0 +#endif + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2025 Victor Zverovich +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // uint32_t, uint64_t + +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief the shortest decimal representation of a double + +A C++11 port of the conversion of Zmij by Victor Zverovich +(https://github.com/vitaut/zmij, MIT license): the shortest decimal in the +rounding interval of a double, the closest one if there are several. Zmij +credits Xiang JunBo (producing the shorter candidate without a division) and +Dougall Johnson (the compressed powers of ten). The powers of ten are taken +from the table for number parsing (pow5_table.hpp) where it holds them, and +computed from the compressed tables of Zmij beyond it. +*/ +namespace zmij +{ + +/// the compressed powers of ten of Zmij +inline const std::array& pow10_minor() noexcept +{ + static const std::array table = + { + { + 0x8000000000000000u, 0xa000000000000000u, 0xc800000000000000u, 0xfa00000000000000u, 0x9c40000000000000u, + 0xc350000000000000u, 0xf424000000000000u, 0x9896800000000000u, 0xbebc200000000000u, 0xee6b280000000000u, + 0x9502f90000000000u, 0xba43b74000000000u, 0xe8d4a51000000000u, 0x9184e72a00000000u, 0xb5e620f480000000u, + 0xe35fa931a0000000u, 0x8e1bc9bf04000000u, 0xb1a2bc2ec5000000u, 0xde0b6b3a76400000u, 0x8ac7230489e80000u, + 0xad78ebc5ac620000u, 0xd8d726b7177a8000u, 0x878678326eac9000u, 0xa968163f0a57b400u, 0xd3c21bcecceda100u, + 0x84595161401484a0u, 0xa56fa5b99019a5c8u, 0xcecb8f27f4200f3au + } + }; + return table; +} + +/// (high, low) pairs +inline const std::array& pow10_major() noexcept +{ + static const std::array table = + { + { + 0xaddcb9e83c6b1793u, 0xdf4abe242a1bbf3eu, 0xaf8e5410288e1b6fu, 0x07ecf0ae5ee44ddau, 0xb1442798f49ffb4au, 0x99cd11cfdf41779du, + 0xb2fe3f0b8599ef07u, 0x861fa7e6dcb4aa15u, 0xb4bca50b065abe63u, 0x0fed077a756b53aau, 0xb67f6455292cbf08u, 0x1a3bc84c17b1d543u, + 0xb84687c269ef3bfbu, 0x3d5d514f40eea742u, 0xba121a4650e4ddebu, 0x92f34d62616ce413u, 0xbbe226efb628afeau, 0x890489f70a55368cu, + 0xbdb6b8e905cb600fu, 0x5400e987bbc1c921u, 0xbf8fdb78849a5f96u, 0xde98520472bdd034u, 0xc16d9a0095928a27u, 0x75b7053c0f178294u, + 0xc350000000000000u, 0x0000000000000000u, 0xc5371912364ce305u, 0x6c28000000000000u, 0xc722f0ef9d80aad6u, 0x424d3ad2b7b97ef6u, + 0xc913936dd571c84cu, 0x03bc3a19cd1e38eau, 0xcb090c8001ab551cu, 0x5cadf5bfd3072cc6u, 0xcd036837130890a1u, 0x36dba887c37a8c10u, + 0xcf02b2c21207ef2eu, 0x94f967e45e03f4bcu, 0xd106f86e69d785c7u, 0xe13336d701beba52u, 0xd31045a8341ca07cu, 0x1ede48111209a051u, + 0xd51ea6fa85785631u, 0x552a74227f3ea566u, 0xd732290fbacaf133u, 0xa97c177947ad4096u, 0xd94ad8b1c7380874u, 0x18375281ae7822bdu, + 0xdb68c2ca82ed2a05u, 0xa67398db9f6820e1u + } + }; + return table; +} + +/// one bit per power: whether the computed value is one unit too large +inline const std::array& pow10_fixups() noexcept +{ + static const std::array table = + { + { + 0x8d8fc810u, 0x06100293u, 0x19000000u, 0x00100000u, 0x00000908u, 0x00000000u, 0x04e00300u, 0x3807e0b2u, 0x3d83d793u, 0x0006f5ccu, + 0x00000000u, 0xffff0000u, 0x8076337du, 0x4ff45ba0u, 0x09405033u, 0x034376d9u, 0x09000000u, 0x4e100501u, 0x076d14dcu, 0xf964f45eu, + 0x0000003du + } + }; + return table; +} + +/// the 128-bit significand of 10^k, rounded down, for k in [-307, 341] +/// (compute_pow10 of Zmij) +inline uint128_parts compute_pow10(int k) noexcept +{ + const auto i = static_cast(k + 307); + const std::uint64_t m = pow10_minor()[(i + 24) % 28]; + const std::size_t j = 2 * static_cast((i + 24) / 28); + const std::uint64_t h_hi = pow10_major()[j]; + const std::uint64_t h_lo = pow10_major()[j + 1]; + const std::uint64_t h1 = full_multiplication(h_lo, m).high; + const std::uint64_t c0 = h_lo * m; + const std::uint64_t c1 = h1 + (h_hi * m); + const std::uint64_t c2 = (c1 < h1 ? 1u : 0u) + full_multiplication(h_hi, m).high; + uint128_parts r{}; + if ((c2 >> 63u) != 0) + { + r.high = c2; + r.low = c1; + } + else + { + r.high = (c2 << 1u) | (c1 >> 63u); + r.low = (c1 << 1u) | (c0 >> 63u); + } + r.low -= (pow10_fixups()[i >> 5u] >> (i & 31u)) & 1u; + return r; +} + +/// The 128-bit significand of 10^k, rounded down, for k in [-342, 341]. +/// Up to 10^308, the table for number parsing holds the same significands +/// (those of 5^k), except for k in [-27, -1], where it holds them one unit +/// larger (as the Eisel-Lemire algorithm needs them). +inline uint128_parts pow10(int k) noexcept +{ + if (k > pow5_128_largest_power) + { + return compute_pow10(k); // (only for the smallest doubles) + } + const auto i = 2 * static_cast(k - pow5_128_smallest_power); + uint128_parts r{pow5_128()[i + 1], pow5_128()[i]}; + const std::uint64_t adjust = static_cast(k + 27) < 27u ? 1u : 0u; + r.high -= r.low < adjust ? 1u : 0u; + r.low -= adjust; + return r; +} + +/// (x_hi * 2^64 + x_lo) * y >> 64, as 128 bits +inline uint128_parts umul192_hi128(std::uint64_t x_hi, std::uint64_t x_lo, std::uint64_t y) noexcept +{ + const uint128_parts p = full_multiplication(x_hi, y); + uint128_parts r{}; + r.low = p.low + full_multiplication(x_lo, y).high; + r.high = p.high + (r.low < p.low ? 1u : 0u); + return r; +} + +/// (x * y + c) >> 64 +inline std::uint64_t umul128_add_hi64(std::uint64_t x, std::uint64_t y, std::uint64_t c) noexcept +{ + const uint128_parts p = full_multiplication(x, y); + return p.high + (p.low + c < p.low ? 1u : 0u); +} + +/// the result of Zmij: the shorter candidate and, if that is outside the +/// rounding interval, the digit after it (16 bytes: returned in registers) +struct shortest_decimal +{ + std::uint64_t integral; ///< the shorter candidate (15 or 16 digits for normal doubles) + int exponent; ///< the decimal exponent of the digit after it + unsigned char digit; ///< the digit after it (if has_digit) + bool has_digit; ///< whether the shortest decimal is integral * 10 + digit +}; + +/// The shortest decimal in the rounding interval of a positive finite double +/// given by its bits, the closest one if there are several (to_decimal of +/// Zmij, which keeps the last digit apart: the 15 or 16 digits before it can be +/// converted without a multiplication by 10 first). Always inlined: GCC +/// otherwise calls it, and its result goes through memory. +JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexcept +{ + constexpr int extra_shift = 9; + const auto raw_exp = static_cast((bits >> 52u) & 0x7FFu); + std::uint64_t bin_sig = bits & ((std::uint64_t{1} << 52u) - 1); + // a power of two has a narrower interval below (except the smallest normal) + const bool regular = bin_sig != 0 || raw_exp <= 1; + const int bin_exp = (raw_exp == 0 ? 1 : raw_exp) - 1075; + if (raw_exp != 0) + { + bin_sig |= std::uint64_t{1} << 52u; + } + // floor(log10(2^bin_exp)), or floor(log10(3/4 * 2^bin_exp)) for the irregular case + const int dec_exp = ((bin_exp * 315653) - (regular ? 0 : 131072)) >> 20; + // scaled by 10^(-dec_exp - 1): the integral part is the shorter candidate + const int shift = bin_exp + ((-(dec_exp + 1) * 217707) >> 16) + 1 + extra_shift; + const uint128_parts p10 = pow10(-dec_exp - 1); + const uint128_parts p = umul192_hi128(p10.high, p10.low, bin_sig << static_cast(shift)); + std::uint64_t integral = p.high >> static_cast(extra_shift); + // (no casts in the shifts: MSVC 2015 warns C4554 on them despite the parentheses) + constexpr unsigned fractional_low_shift = static_cast(extra_shift); + constexpr unsigned fractional_high_shift = 64u - fractional_low_shift; + const std::uint64_t fractional = (p.high << fractional_high_shift) | (p.low >> fractional_low_shift); + std::uint64_t digit = 0; + bool round_up = false; + bool round_down = false; + if (JSON_HEDLEY_LIKELY(regular)) + { + const std::uint64_t half_ulp = (p10.high >> static_cast(extra_shift + 1 - shift)) + (1 - (bin_sig & 1u)); + round_up = fractional + half_ulp < fractional; + round_down = half_ulp > fractional; + // the last digit of the longer candidate, rounded to nearest + digit = umul128_add_hi64(fractional, 10, (std::uint64_t{1} << 63u) + 6); + if (fractional == (std::uint64_t{1} << 62u)) + { + digit = 2; // 2.5 rounds to 2 + } + } + else + { + const std::uint64_t half_ulp = p10.high >> static_cast(extra_shift + 1 - shift); + round_up = half_ulp > ~std::uint64_t{0} - fractional; + round_down = (half_ulp >> 1u) > fractional; + digit = umul128_add_hi64(fractional, 10, (std::uint64_t{1} << 63u) - 1); + const std::uint64_t lowest = umul128_add_hi64(fractional - (half_ulp >> 1u), 10, ~std::uint64_t{0}); + digit = digit < lowest ? lowest : digit; + } + integral += round_up ? 1u : 0u; + // if the shorter candidate is outside the rounding interval: one digit more + return shortest_decimal{integral, dec_exp, static_cast(digit), !round_up && !round_down}; +} + +} // namespace zmij +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include @@ -28402,6 +29834,381 @@ inline char* format_buffer(char* buf, int len, int decimal_exponent, return append_exponent(buf, n - 1); } +/// eight decimal digits (a value below 10^8) as bytes 0..9, the first digit +/// in the most significant byte: three steps that divide all lanes at once +/// by a multiplication (the conversion of Xiang JunBo, as in Zmij) +inline std::uint64_t eight_digit_bytes(std::uint64_t abcdefgh) noexcept +{ + const std::uint64_t abcd_efgh = abcdefgh + (((std::uint64_t{1} << 32u) - 10000u) * ((abcdefgh * (((std::uint64_t{1} << 40u) / 10000u) + 1u)) >> 40u)); + const std::uint64_t ab_cd_ef_gh = abcd_efgh + (((std::uint64_t{1} << 16u) - 100u) * (((abcd_efgh * (((std::uint64_t{1} << 19u) / 100u) + 1u)) >> 19u) & 0x7F0000007Fu)); + return ab_cd_ef_gh + (((std::uint64_t{1} << 8u) - 10u) * (((ab_cd_ef_gh * (((std::uint64_t{1} << 10u) / 10u) + 1u)) >> 10u) & 0x000F000F000F000Fu)); +} + +/// store the bytes of v, the most significant one first (one byte swap and +/// one store where the byte order is known: compilers do not reliably merge +/// the byte stores once this is inlined) +inline void store_msb_first(char* p, std::uint64_t v) noexcept +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + v = __builtin_bswap64(v); + std::memcpy(p, &v, sizeof(v)); +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + std::memcpy(p, &v, sizeof(v)); +#elif defined(_MSC_VER) // (little-endian on all its targets) + v = _byteswap_uint64(v); + std::memcpy(p, &v, sizeof(v)); +#else + for (unsigned i = 0; i < 8; ++i) + { + p[i] = static_cast(v >> (56u - (8u * i))); + } +#endif +} + +/*! +@brief digits * 10^exp for a double, in the layout of format_buffer() + +The layout is that of format_buffer() with min_exp -4 and max_exp 15 (the +digits10 of double). The digits are converted eight at a time and placed +with fixed-size moves instead of per-digit loops and moves of the buffer. + +@param[in] digits the digits (not 0, at most 17 digits; trailing zeros allowed) +@param[in] exp the decimal exponent of the last digit +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_decimal(char* first, std::uint64_t digits, int exp) noexcept +{ + JSON_ASSERT(digits != 0 && digits < 100000000000000000u); + const std::uint64_t upper = digits / 100000000u; + const std::uint64_t b0 = upper / 100000000u; // (one digit: it is its own byte) + const std::uint64_t b1 = eight_digit_bytes(upper % 100000000u); + const std::uint64_t b2 = eight_digit_bytes(digits % 100000000u); + // leading and trailing zero digits: zero bytes, counted without division + int leading = 16; + int zeros = 16; + if (b0 != 0) + { + leading = count_leading_zeros(b0) / 8; + } + else if (b1 != 0) + { + leading = 8 + (count_leading_zeros(b1) / 8); + } + else + { + leading += count_leading_zeros(b2) / 8; + } + if (b2 != 0) + { + zeros = count_trailing_zeros(b2) / 8; + } + else if (b1 != 0) + { + zeros = 8 + (count_trailing_zeros(b1) / 8); + } + // (else: 16, b0 is the one digit that is not 0) + // the digits as text at text + leading, then '0's, so that fixed-size + // moves need not check how many digits there are + std::array text; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + store_msb_first(text.data(), b0 + 0x3030303030303030u); + store_msb_first(text.data() + 8, b1 + 0x3030303030303030u); + store_msb_first(text.data() + 16, b2 + 0x3030303030303030u); + std::memset(text.data() + 24, '0', 40); + const int k = 24 - leading - zeros; // significant digits + const int n = k + exp + zeros; // position of the decimal point after the first digit + const char* const s0 = text.data() + leading; + + if (-4 < n && n <= 15) + { + // "0.[000]digits" (n <= 0) is the digits after 1 - n leading '0's + // with the point after the first; "digits[000].0" (n >= k) and + // "dig.its" put the point after n characters + const int pad = n <= 0 ? 1 - n : 0; + const char* const s = s0 - pad; + const int len = k + pad; + const int point = n + pad; + std::memcpy(first, s, 16); + std::memcpy(first + point + 1, s + point, 24); + first[point] = '.'; + return first + (point >= len ? point + 2 : len + 1); + } + + // d.igitse+XX, with at least two exponent digits (as append_exponent()) + std::memcpy(first, s0, 16); + std::memcpy(first + 2, s0 + 1, 16); + first[1] = '.'; + char* const end = first + (k == 1 ? 1 : k + 1); + const int e = n - 1; + const auto ea = e < 0 ? 0u - static_cast(e) : static_cast(e); // (unsigned: no signed overflow to assume) + const bool three = ea >= 100; + end[0] = 'e'; + end[1] = e < 0 ? '-' : '+'; + end[2] = static_cast('0' + (three ? ea / 100 : (ea / 10) % 10)); + end[3] = static_cast('0' + (three ? (ea / 10) % 10 : ea % 10)); + end[4] = static_cast('0' + (ea % 10)); + return end + (three ? 5 : 4); +} + +/*! +@brief the shortest decimal of a positive double (Zmij), as write_decimal() +writes it + +For a normal double, the shorter candidate has 15 or 16 digits: they are +converted at once (two halves of eight digits) and followed by the digit +after them, if there is one, without the multiplication and division by 10 +that counting the digits of one number would take. The fixed layouts move +the digits after the point by one byte. + +@return a pointer past the text; up to 41 bytes at @a first are written + (some beyond the returned end) +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* write_shortest(char* first, const zmij::shortest_decimal d) noexcept +{ + const std::uint64_t sig = d.integral; + if (JSON_HEDLEY_UNLIKELY(sig < 100000000000000u || sig >= 10000000000000000u)) + { + // (subnormals) + return d.has_digit ? write_decimal(first, (sig * 10) + d.digit, d.exponent) : write_decimal(first, sig, d.exponent + 1); + } + const bool sixteen = sig >= 1000000000000000u; // (else 15 digits) + const int last = d.has_digit ? d.digit : 0; + const std::uint64_t upper = sig / 100000000u; +#if JSON_DTOA_SSE2 + // NOLINTBEGIN(portability-simd-intrinsics) + // the two halves in the 64-bit lanes, each as abcd * 2^32 + efgh, then as + // bytes (as eight_digit_bytes(), one lane each) + const __m128i x = _mm_set_epi64x(static_cast(sig - (upper * 100000000u)), static_cast(upper)); // NOLINT(runtime/int) + const __m128i abcd = _mm_srli_epi64(_mm_mul_epu32(x, _mm_set1_epi64x(109951163)), 40); // 2^40 / 10000 + 1 + const __m128i abcd_efgh = _mm_add_epi64(x, _mm_mul_epu32(abcd, _mm_set1_epi64x(4294957296))); // 2^32 - 10000 + // 32-bit lanes in the order of the text: abcd, efgh of both halves + const __m128i fours = _mm_shuffle_epi32(abcd_efgh, _MM_SHUFFLE(2, 3, 0, 1)); + const __m128i ab = _mm_srli_epi16(_mm_mulhi_epu16(fours, _mm_set1_epi32(5243)), 3); + const __m128i ab_cd = _mm_or_si128(_mm_slli_epi32(_mm_sub_epi16(fours, _mm_mullo_epi16(ab, _mm_set1_epi32(100))), 16), ab); + // 16-bit lanes ab (< 100) -> bytes a, b: 256 * ab - 2559 * (ab / 10) + const __m128i bytes = _mm_sub_epi16(_mm_slli_epi16(ab_cd, 8), _mm_mullo_epi16(_mm_set1_epi16(2559), _mm_mulhi_epu16(ab_cd, _mm_set1_epi16(6554)))); + // the last digit that is not 0 (sig is not 0) + const auto nonzero = static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8(bytes, _mm_setzero_si128()))); + const int digits = 63 - count_leading_zeros(nonzero) + (sixteen ? 1 : 0); // without trailing zeros + const __m128i chars = _mm_add_epi8(bytes, _mm_set1_epi8('0')); + // the 16 characters from the first digit + const __m128i s = sixteen ? chars : _mm_or_si128(_mm_srli_si128(chars, 1), _mm_slli_si128(_mm_cvtsi32_si128('0' + last), 15)); + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [&s](char* p) noexcept + { + std::memcpy(p, &s, 16); + }; + const char first_digit = static_cast(_mm_cvtsi128_si32(s)); + // NOLINTEND(portability-simd-intrinsics) +#elif JSON_DTOA_NEON + // as with SSE2: the halves in 32-bit lanes, then abcd, efgh of both + const uint32x2_t halves = vcreate_u32(upper | ((sig - (upper * 100000000u)) << 32u)); + const uint32x2_t abcd = vmovn_u64(vshrq_n_u64(vmull_n_u32(halves, static_cast(((std::uint64_t{1} << 40u) / 10000u) + 1u)), 40)); + const uint32x2_t efgh = vmls_n_u32(halves, abcd, 10000u); + const uint32x4_t fours = vcombine_u32(vzip1_u32(abcd, efgh), vzip2_u32(abcd, efgh)); + const uint32x4_t ab = vshrq_n_u32(vmulq_n_u32(fours, 5243u), 19); + const uint16x8_t ab_cd = vreinterpretq_u16_u32(vorrq_u32(ab, vshlq_n_u32(vmlsq_n_u32(fours, ab, 100u), 16))); + const uint16x8_t tens = vshrq_n_u16(vmulq_n_u16(ab_cd, 103u), 10); + const uint8x16_t bytes = vreinterpretq_u8_u16(vorrq_u16(tens, vshlq_n_u16(vmlsq_n_u16(ab_cd, tens, 10u), 8))); + // the last digit that is not 0 (sig is not 0): a nibble per byte + const std::uint64_t nonzero = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vtstq_u8(bytes, bytes)), 4)), 0); + const int digits = ((63 - count_leading_zeros(nonzero)) / 4) + (sixteen ? 1 : 0); // without trailing zeros + const uint8x16_t chars = vaddq_u8(bytes, vdupq_n_u8('0')); + // the 16 characters from the first digit + const uint8x16_t s = sixteen ? chars : vextq_u8(chars, vdupq_n_u8(static_cast('0' + last)), 1); + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [&s](char* p) noexcept + { + vst1q_u8(reinterpret_cast(p), s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + }; + const auto first_digit = static_cast(vgetq_lane_u8(s, 0)); +#else + const std::uint64_t hi = eight_digit_bytes(upper); + const std::uint64_t lo = eight_digit_bytes(sig - (upper * 100000000u)); + // trailing zero digits: zero bytes (sig is not 0) + const int zeros = lo != 0 ? count_trailing_zeros(lo) / 8 : 8 + (count_trailing_zeros(hi) / 8); + const int digits = 15 - zeros + (sixteen ? 1 : 0); // without trailing zeros + // the 16 characters from the first digit + const std::uint64_t s_hi = (sixteen ? hi : (hi << 8u) | (lo >> 56u)) + 0x3030303030303030u; + const std::uint64_t s_lo = (sixteen ? lo : (lo << 8u) | static_cast(last)) + 0x3030303030303030u; + const char s16 = static_cast(sixteen ? '0' + last : '0'); // the 17th + const auto store_16 = [s_hi, s_lo](char* p) noexcept + { + store_msb_first(p, s_hi); + store_msb_first(p + 8, s_lo); + }; + const auto first_digit = static_cast(s_hi >> 56u); +#endif + const int len = d.has_digit ? 16 + (sixteen ? 1 : 0) : digits; // significant digits + const int n = 16 + (sixteen ? 1 : 0) + d.exponent; // digits before the point + + if (JSON_HEDLEY_LIKELY(n >= 1 && n <= 15)) + { + // "dig.its" and "digits[000].0": the digits after the point move by + // one byte ('0's follow the digits) +#if JSON_DTOA_SSE2 + // NOLINTBEGIN(portability-simd-intrinsics) + // (in the register: reading the digits back from memory right after + // storing them waits until the stores are done) + const __m128i at = _mm_set1_epi8(static_cast(n)); + const __m128i index = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const __m128i before = _mm_cmpgt_epi8(at, index); + const __m128i after = _mm_cmpgt_epi8(index, at); + const __m128i text = _mm_or_si128(_mm_or_si128(_mm_and_si128(s, before), _mm_and_si128(_mm_slli_si128(s, 1), after)), + _mm_andnot_si128(_mm_or_si128(before, after), _mm_set1_epi8('.'))); + std::memcpy(first, &text, 16); + first[16] = static_cast(_mm_extract_epi16(s, 7) >> 8); + first[17] = s16; + // NOLINTEND(portability-simd-intrinsics) +#elif JSON_DTOA_NEON + const uint8x16_t index = vcombine_u8(vcreate_u8(0x0706050403020100u), vcreate_u8(0x0F0E0D0C0B0A0908u)); + const uint8x16_t at = vdupq_n_u8(static_cast(n)); + const uint8x16_t after_point = vbslq_u8(vcgtq_u8(index, at), vextq_u8(vdupq_n_u8(0), s, 15), vdupq_n_u8('.')); + vst1q_u8(reinterpret_cast(first), vbslq_u8(vcltq_u8(index, at), s, after_point)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + first[16] = static_cast(vgetq_lane_u8(s, 15)); + first[17] = s16; +#else + store_16(first); + first[16] = s16; + std::uint64_t after_point[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(after_point, first + n, 16); + std::memcpy(first + n + 1, after_point, 16); + first[n] = '.'; +#endif + return first + (n >= len ? n + 2 : len + 1); + } + if (n <= 0 && n > -4) + { + // "0.[000]digits" + std::memset(first, '0', 8); + first[1] = '.'; + store_16(first + 2 - n); + first[18 - n] = s16; + return first + 2 - n + len; + } + // d.igitse+XX, with at least two exponent digits (as append_exponent()) + store_16(first + 1); + first[17] = s16; + first[0] = first_digit; + first[1] = '.'; + char* const end = first + (len == 1 ? 1 : len + 1); + const int e = n - 1; + const auto ea = e < 0 ? 0u - static_cast(e) : static_cast(e); // (unsigned: no signed overflow to assume) + const bool three = ea >= 100; + end[0] = 'e'; + end[1] = e < 0 ? '-' : '+'; + end[2] = static_cast('0' + (three ? ea / 100 : (ea / 10) % 10)); + end[3] = static_cast('0' + (three ? (ea / 10) % 10 : ea % 10)); + end[4] = static_cast('0' + (ea % 10)); + return end + (three ? 5 : 4); +} + +/*! +@brief whether FloatType is an IEEE 754 binary64 type (a double, or a long double +that has the same format, as with MSVC and on Apple's Arm CPUs) + +These are the types the conversion of Zmij (see zmij.hpp) is used for; all +others (binary32, or a format the library does not know) use Grisu2. +*/ +template +constexpr bool has_binary64_format() noexcept +{ + return std::numeric_limits::is_iec559 + && std::numeric_limits::digits == 53 + && std::numeric_limits::max_exponent == 1024 + && sizeof(FloatType) == sizeof(std::uint64_t); +} + +template +struct is_binary64 : std::integral_constant()> {}; + +/// a positive finite float (other than binary64): Grisu2 and format_buffer() +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive_grisu2(char* first, const char* last, FloatType value) +{ + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); + static_cast(last); // (only used in the assertion) + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e., the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + grisu2(first, len, decimal_exponent, value); + + JSON_ASSERT(len <= std::numeric_limits::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits::digits10; + + JSON_ASSERT(last - first >= kMaxExp + 2); + JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); + + return format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +/// a positive finite binary64 number: the shortest digits (Zmij), laid out by +/// write_shortest() (through a local buffer if [first, last) is shorter than +/// the 41 bytes it may write) +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive_zmij(char* first, const char* last, FloatType value) +{ + static_assert(is_binary64::value, + "internal error: the conversion of Zmij needs IEEE 754 binary64 numbers"); + std::uint64_t bits = 0; + std::memcpy(&bits, &value, sizeof(bits)); + const zmij::shortest_decimal d = zmij::to_shortest(bits); + if (JSON_HEDLEY_LIKELY(last - first >= 41)) + { + return write_shortest(first, d); + } + std::array buf; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + const auto len = static_cast(write_shortest(buf.data(), d) - buf.data()); + JSON_ASSERT(last - first >= static_cast(len)); + std::memcpy(first, buf.data(), len); + return first + len; +} + +/// a positive finite binary64 number: Zmij (as a long double has the format of +/// a double here, its bits are those of the double of the same value) +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value, std::true_type /*is_binary64*/) +{ + return write_positive_zmij(first, last, value); +} + +/// a positive finite float of any other format: Grisu2 +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value, std::false_type /*is_binary64*/) +{ + return write_positive_grisu2(first, last, value); +} + +/// a positive finite float: Zmij for binary64 numbers, Grisu2 otherwise +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* write_positive(char* first, const char* last, FloatType value) +{ + return write_positive(first, last, value, is_binary64 {}); +} + } // namespace dtoa_impl /*! @@ -28419,7 +30226,6 @@ JSON_HEDLEY_NON_NULL(1, 2) JSON_HEDLEY_RETURNS_NON_NULL char* to_chars(char* first, const char* last, FloatType value) { - static_cast(last); // maybe unused - fix warning JSON_ASSERT(std::isfinite(value)); // Use signbit(value) instead of (value < 0) since signbit works for -0. @@ -28445,28 +30251,7 @@ char* to_chars(char* first, const char* last, FloatType value) JSON_HEDLEY_DIAGNOSTIC_POP #endif - JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); - - // Compute v = buffer * 10^decimal_exponent. - // The decimal digits are stored in the buffer, which needs to be interpreted - // as an unsigned decimal integer. - // len is the length of the buffer, i.e., the number of decimal digits. - int len = 0; - int decimal_exponent = 0; - dtoa_impl::grisu2(first, len, decimal_exponent, value); - - JSON_ASSERT(len <= std::numeric_limits::max_digits10); - - // Format the buffer like printf("%.*g", prec, value) - constexpr int kMinExp = -4; - // Use digits10 here to increase compatibility with version 2. - constexpr int kMaxExp = std::numeric_limits::digits10; - - JSON_ASSERT(last - first >= kMaxExp + 2); - JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); - JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); - - return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); + return dtoa_impl::write_positive(first, last, value); } } // namespace detail @@ -28508,6 +30293,12 @@ class serializer using number_float_t = typename BasicJsonType::number_float_t; using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type of the absolute value of an integer: number_unsigned_t, or its + /// unsigned counterpart if it is signed (it may be number_integer_t, #2573) + using number_magnitude_t = typename std::conditional < + std::is_integral::value && std::is_signed::value, + std::make_unsigned, + std::enable_if>::type::type; using binary_char_t = typename BasicJsonType::binary_t::value_type; public: @@ -29231,6 +31022,33 @@ class serializer break; } } + + // A byte below 0x60 is a complete code point and needs no + // UTF-8 decoding. The scanner stops only at a control + // character, a quote, or a backslash in that range, so look up + // its escape sequence and write all 8 bytes of the table entry + // straight into the write buffer; only the length stored in the + // last byte of the entry is kept. + if (data[i] < 0x60) + { + // emit any bytes still pending in string_buffer first to + // preserve output order + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + const auto& escape = ascii_escapes()[data[i]]; + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size()); + write_buffer_pos += static_cast(escape[7]); + bytes_after_last_accept = 0; + undumped_chars = 0; + continue; + } } const auto byte = static_cast(s[i]); @@ -29239,92 +31057,39 @@ class serializer { case UTF8_ACCEPT: // decode found a new code point { - switch (codepoint) + // the code point is at least 0x60 (smaller ones are handled + // above) and is not printable ASCII (which the scanner + // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII + // character + if (EnsureAscii) { - case 0x08: // backspace + JSON_ASSERT(codepoint >= 0x7F); + if (error_handler == error_handler_t::keep) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; + // this character was buffered as raw bytes below + // in case it turned out to be part of an + // ill-formed sequence (which is kept as is); now + // that it decoded to a well-formed code point, + // undo that and \u-escape it like any other + // character instead + bytes = bytes_after_last_accept; } - - case 0x09: // horizontal tab + if (codepoint <= 0xFFFF) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; + write_u_escape(bytes, static_cast(codepoint)); } - - case 0x0A: // newline + else { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // EnsureAscii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) - { - if (EnsureAscii && error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes - // below in case it turned out to be part of - // an ill-formed sequence (which is kept as - // is); now that it decoded to a well-formed - // code point, undo that and \u-escape it - // like any other character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; + write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); + write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); } } + else + { + // copy byte to buffer (all previous bytes of the + // sequence have been copied in the default case below) + string_buffer[bytes++] = s[i]; + } // write buffer and reset index; there must be 13 bytes // left, as this is the maximal number of bytes to be @@ -29686,7 +31451,7 @@ class serializer @param[in] x unsigned integer number to count its digits @return number of decimal digits */ - unsigned int count_digits(number_unsigned_t x) noexcept + unsigned int count_digits(number_magnitude_t x) noexcept { unsigned int n_digits = 1; for (;;) @@ -29712,6 +31477,121 @@ class serializer } } + /*! + @brief the JSON form of each character below 0x60 + + Entry c holds the escape sequence of character c - the short form for + \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters - + or c itself if it needs no escaping, padded with zeros to 7 bytes. The last + byte is the length of the sequence. The 8 bytes of an entry are copied at + once, so writing an escape needs neither a branch nor a byte-wise loop. + */ + static const std::array, 0x60>& ascii_escapes() noexcept + { + static const std::array, 0x60> escapes = + { + { + {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00 + {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01 + {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02 + {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03 + {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04 + {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05 + {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06 + {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07 + {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08 + {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09 + {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A + {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B + {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C + {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D + {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E + {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F + {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10 + {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11 + {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12 + {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13 + {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14 + {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15 + {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16 + {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17 + {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18 + {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19 + {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A + {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B + {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C + {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D + {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E + {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F + {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20 + {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21 + {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22 + {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23 + {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24 + {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25 + {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26 + {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27 + {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28 + {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29 + {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A + {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B + {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C + {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D + {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E + {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F + {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30 + {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31 + {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32 + {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33 + {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34 + {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35 + {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36 + {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37 + {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38 + {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39 + {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A + {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B + {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C + {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D + {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E + {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F + {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40 + {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41 + {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42 + {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43 + {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44 + {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45 + {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46 + {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47 + {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48 + {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49 + {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A + {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B + {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C + {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D + {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E + {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F + {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50 + {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51 + {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52 + {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53 + {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54 + {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55 + {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56 + {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57 + {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58 + {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59 + {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A + {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B + {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C + {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D + {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E + {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F + } + }; + return escapes; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * @@ -29825,8 +31705,9 @@ class serializer /*! @brief dump an integer - Dump a given integer, appending it to @ref write_buffer. Works internally with - @a number_buffer. + Dump a given integer, appending it to @ref write_buffer (directly: copying + the digits from another buffer right after writing them waits until the + stores are done). @param[in] x integer number (signed or unsigned) to dump @tparam NumberType either @a number_integer_t or @a number_unsigned_t @@ -29861,33 +31742,57 @@ class serializer return; } - // use a pointer to fill the buffer - auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto) + // use a pointer to fill the buffer (room for as much as number_buffer holds) + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + number_buffer.size() > write_buffer.size())) + { + flush(); + } + auto* buffer_ptr = write_buffer.data() + write_buffer_pos; - number_unsigned_t abs_value; + number_magnitude_t abs_value; - unsigned int n_chars{}; + // one byte for the minus sign + unsigned int n_chars = 0; if (is_negative_number(x)) { *buffer_ptr = '-'; abs_value = remove_sign(static_cast(x)); - - // account one more byte for the minus sign - n_chars = 1 + count_digits(abs_value); + n_chars = 1; } else { - abs_value = static_cast(x); - n_chars = count_digits(abs_value); + abs_value = static_cast(x); } + // up to 16 digits: eight at a time (as the digits of floats), written + // without leading zeros + if (abs_value < 10000000000000000u) + { + const std::uint64_t value = abs_value; + const std::uint64_t upper = value / 100000000u; + const std::uint64_t first = dtoa_impl::eight_digit_bytes(upper != 0 ? upper : value); + const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) + char* const p = buffer_ptr + n_chars; + dtoa_impl::store_msb_first(p, (first << (8 * leading)) + 0x3030303030303030u); + n_chars += 8 - leading; + if (upper != 0) + { + dtoa_impl::store_msb_first(p + 8 - leading, dtoa_impl::eight_digit_bytes(value - (upper * 100000000u)) + 0x3030303030303030u); + n_chars += 8; + } + write_buffer_pos += n_chars; + return; + } + + n_chars += count_digits(abs_value); + // spare 1 byte for '\0' JSON_ASSERT(n_chars < number_buffer.size() - 1); // jump to the end to generate the string from backward, // so we later avoid reversing the result - buffer_ptr += static_cast(n_chars); + buffer_ptr += n_chars; // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu // See: https://www.youtube.com/watch?v=o4-CwDo2zpg @@ -29910,14 +31815,13 @@ class serializer *(--buffer_ptr) = static_cast('0' + abs_value); } - put_buffer(number_buffer, n_chars); + write_buffer_pos += n_chars; } /*! @brief dump a floating-point number - Dump a given floating-point number, appending it to @ref write_buffer. Works internally - with @a number_buffer. + Dump a given floating-point number, appending it to @ref write_buffer. @param[in] x floating-point number to dump */ @@ -29944,10 +31848,15 @@ class serializer void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) { - auto* begin = number_buffer.data(); + // directly into the write buffer: copying the text from number_buffer + // right after to_chars() wrote it waits until its stores are done + if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + number_buffer.size() > write_buffer.size())) + { + flush(); + } + auto* begin = write_buffer.data() + write_buffer_pos; auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - - put_buffer(number_buffer, static_cast(end - begin)); + write_buffer_pos += static_cast(end - begin); } JSON_HEDLEY_NON_NULL(1) @@ -30021,17 +31930,6 @@ class serializer } } - /* - * Overload to make the compiler happy while it is instantiating - * dump_integer for number_unsigned_t. - * Must never be called. - */ - number_unsigned_t remove_sign(number_unsigned_t x) - { - JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE - return x; // LCOV_EXCL_LINE - } - /* * Helper function for dump_integer * @@ -30041,10 +31939,10 @@ class serializer * absolute values of INT_MIN and INT_MAX are usually not the same. See * #1708 for details. */ - number_unsigned_t remove_sign(number_integer_t x) noexcept + number_magnitude_t remove_sign(number_integer_t x) noexcept { JSON_ASSERT(x < 0); - return static_cast(-(x + 1)) + 1; + return static_cast(-(x + 1)) + 1; } private: @@ -30088,7 +31986,7 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include - +// IWYU pragma: export // #include // __ _____ _____ _____ // __| | __| | | | JSON for Modern C++ @@ -30185,6 +32083,15 @@ private: return self.end(); } + /// @brief throw the exception of a failed at() lookup + /// @note kept out of line so that at_impl() stays small enough to inline + /// @throw std::out_of_range always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_key_not_found() + { + JSON_THROW(std::out_of_range("key not found")); + } + /// @brief shared implementation of the const and non-const at() overloads /// @throw std::out_of_range if @a key is not found template @@ -30193,7 +32100,7 @@ private: const auto it = find_impl(self, key); if (it == self.end()) { - JSON_THROW(std::out_of_range("key not found")); + throw_key_not_found(); } return it->second; } @@ -30238,6 +32145,17 @@ public: return {std::prev(this->end()), true}; } + /// @brief add an element at the end without checking whether the key exists + /// @note The caller must guarantee that @a key is not yet contained, otherwise + /// the container holds duplicate keys. This avoids the linear search of + /// emplace() and operator[] when many keys known to be new are added. + template::value, int> = 0> + void emplace_new_key(const key_type& key, V && t) + { + append(key, std::forward(t)); + } + T& operator[](const key_type& key) { return emplace(key, T{}).first->second; @@ -30928,8 +32846,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec "NumberUnsignedType must be at least as wide as NumberIntegerType, " "because it has to hold the absolute value of every NumberIntegerType value"); + // NumberUnsignedType may be the same type as NumberIntegerType (#2573), but + // a repeated NumberFloatType would give overloads that collide, so it gets + // a readable diagnostic instead. + static_assert(!std::is_same::value && !std::is_same::value, + "NumberFloatType must be a different type than NumberIntegerType and NumberUnsignedType"); + private: + /// the kind a value of kind @a t is stored as: if number_unsigned_t is the + /// same type as number_integer_t, every integer is stored as number_integer + /// (#2573); every other kind is stored as itself + static constexpr value_t stored_kind(const value_t t) noexcept + { + return (std::is_same::value && t == value_t::number_unsigned) ? value_t::number_integer : t; + } + /// helper for exception-safe object creation template JSON_HEDLEY_RETURNS_NON_NULL @@ -31003,8 +32935,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec json_value(boolean_t v) noexcept : boolean(v) {} /// constructor for numbers (integer) json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (unsigned); takes a placeholder if number_unsigned_t is number_integer_t + json_value(detail::number_unsigned_overload_type v) noexcept : number_unsigned(v) {} /// constructor for numbers (floating-point) json_value(number_float_t v) noexcept : number_float(v) {} /// constructor for empty values of a given type @@ -31013,26 +32945,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec switch (t) { case value_t::object: - { - object = create(); - break; - } - case value_t::array: - { - array = create(); - break; - } - case value_t::string: - { - string = create(""); - break; - } - case value_t::binary: { - binary = create(); + // allocating: kept out of line, so the scalar cases below inline + init_owned(t); break; } @@ -31072,13 +32990,41 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec object = nullptr; // silence warning, see #821 if (JSON_HEDLEY_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(detail::exception_id::internal_error, "961c151d2e87f2686a955a9be24d316f1362bf21 3.12.0", nullptr)); // LCOV_EXCL_LINE + detail::throw_literal(detail::exception_id::internal_error, "961c151d2e87f2686a955a9be24d316f1362bf21 3.12.0", nullptr); // LCOV_EXCL_LINE } break; } } } + JSON_HEDLEY_NEVER_INLINE + void init_owned(value_t t) + { + switch (t) + { + case value_t::object: + object = create(); + break; + case value_t::array: + array = create(); + break; + case value_t::string: + string = create(""); + break; + case value_t::binary: + binary = create(); + break; + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + break; + } + } + /// constructor for strings json_value(const string_t& value) : string(create(value)) {} @@ -31148,6 +33094,34 @@ private: } } + // true if no child of the non-empty array/object v has children of + // its own, i.e. v can be freed in one go (its destructor then + // destroys scalars and empty containers only, so it recurses at most + // one level deep); stops at the first structured child it meets + static bool all_children_leaves(const basic_json& v) noexcept + { + if (v.m_data.m_type == value_t::array) + { + const auto& a = *v.m_data.m_value.array; + for (auto it = a.rbegin(); it != a.rend(); ++it) + { + if (!has_no_children(*it)) + { + return false; + } + } + return true; + } + for (const auto& el : *v.m_data.m_value.object) + { + if (!has_no_children(el.second)) + { + return false; + } + } + return true; + } + // The walk in destroy_container() may take the children of a // container in any order, as long as it picks the same child again // while that container is not modified in between. Arrays and @@ -31203,15 +33177,16 @@ private: } } - // deallocates the (already empty) array/object held by v; this is - // the same allocator-based free the old recursive implementation - // used, just factored out so every level of the walk in destroy() - // can share it + // deallocates the array/object held by v, which is either empty or + // holds only scalars and empty containers (so its destructor + // recurses at most one level deep); this is the same allocator-based + // free the old recursive implementation used, just factored out so + // every level of the walk in destroy() can share it static void free_container(basic_json& v) noexcept { + JSON_ASSERT(has_no_children(v) || all_children_leaves(v)); if (v.m_data.m_type == value_t::array) { - JSON_ASSERT(v.m_data.m_value.array->empty()); AllocatorType alloc; std::allocator_traits::destroy(alloc, v.m_data.m_value.array); std::allocator_traits::deallocate(alloc, v.m_data.m_value.array, 1); @@ -31219,7 +33194,6 @@ private: else { JSON_ASSERT(v.m_data.m_type == value_t::object); - JSON_ASSERT(v.m_data.m_value.object->empty()); AllocatorType alloc; std::allocator_traits::destroy(alloc, v.m_data.m_value.object); std::allocator_traits::deallocate(alloc, v.m_data.m_value.object, 1); @@ -31264,40 +33238,99 @@ public: return; } - // Destroy the tree without recursing per nesting level and - // without any heap allocation: a heap-allocated flattening - // stack (the previous implementation) can itself throw - // bad_alloc, which would escape this noexcept destructor and - // terminate the program (#5135). - // - // Instead, walk down a chain of children (always the one - // walk_child() picks), reversing links as we go: cur is the - // container currently being emptied, and prev is its parent - // (value_t::null when there is none). Each parent's - // walk_child() slot doubles as storage for that parent's own - // parent link while we are below it, so no extra memory is - // needed; the parent is not modified meanwhile, so - // walk_child() finds that same slot again on the way up. We - // only ever remove a child once it is a scalar or an empty - // array/object, which neither allocates nor recurses more - // than one level deep. - // - // This json_value is not itself a basic_json, so the - // top-level container is first moved into a local stand-in - // ("cur"); a default-constructed basic_json has a null - // pointer in its m_value (see data::m_value's initializer), - // so swapping it with *this leaves this union's own pointer - // null, and it is never looked at or freed a second time. + // This json_value is not itself a basic_json, so the top-level + // container is first moved into a local stand-in ("cur"); a + // default-constructed basic_json has a null pointer in its + // m_value (see data::m_value's initializer), so swapping it + // with *this leaves this union's own pointer null, and it is + // never looked at or freed a second time. basic_json cur; cur.m_data.m_type = t; using std::swap; swap(cur.m_data.m_value, *this); + destroy_bounded(cur, 0); + } + + // Destroy the array/object v, whose children are destroyed first. + // The first recursion_depth_limit() levels are destroyed by plain + // recursion, which is the fastest way and cannot throw; anything + // nested deeper is handed to destroy_walk(), which uses no call stack. + static void destroy_bounded(basic_json& v, std::size_t depth) noexcept + { + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + destroy_walk(v); + return; + } + + if (v.m_data.m_type == value_t::array) + { + for (auto& child : *v.m_data.m_value.array) + { + if (child.m_data.m_type == value_t::array || child.m_data.m_type == value_t::object) + { + destroy_bounded(child, depth + 1); + } + } + } + else + { + for (auto& child : *v.m_data.m_value.object) + { + if (child.second.m_data.m_type == value_t::array || child.second.m_data.m_type == value_t::object) + { + destroy_bounded(child.second, depth + 1); + } + } + } + + // only leaves and already destroyed (null) children remain + free_container(v); + } + + // Destroy the tree below top without recursing per nesting level and + // without any heap allocation: a heap-allocated flattening + // stack (the previous implementation) can itself throw + // bad_alloc, which would escape this noexcept destructor and + // terminate the program (#5135). + // + // Instead, walk down a chain of children (always the one + // walk_child() picks), reversing links as we go: cur is the + // container currently being emptied, and prev is its parent + // (value_t::null when there is none). Each parent's + // walk_child() slot doubles as storage for that parent's own + // parent link while we are below it, so no extra memory is + // needed; the parent is not modified meanwhile, so + // walk_child() finds that same slot again on the way up. We + // only ever remove a child once it is a scalar or an empty + // array/object, which neither allocates nor recurses more + // than one level deep. + // + // This json_value is not itself a basic_json, so the + // top-level container is first moved into a local stand-in + // ("cur"); a default-constructed basic_json has a null + // pointer in its m_value (see data::m_value's initializer), + // so swapping it with *this leaves this union's own pointer + // null, and it is never looked at or freed a second time. + static void destroy_walk(basic_json& top) noexcept + { + basic_json cur; + take(cur, top); + basic_json prev; // value_t::null: no parent + // true right after cur was entered: its children have not been + // looked at yet. If none of them has children of its own, cur is + // freed in one go instead of child by child; this is tried once + // per container, so the walk stays linear. + bool fresh = true; + while (true) { - if (has_no_children(cur)) + const bool fresh_now = fresh; + fresh = false; + if (has_no_children(cur) || (fresh_now && all_children_leaves(cur))) { if (prev.m_data.m_type == value_t::null) { @@ -31336,10 +33369,22 @@ public: take(cur_child_ref, prev); take(prev, cur); take(cur, tmp); + fresh = true; } } + // keep the check whether t owns memory inline so destroying a scalar + // is a compare and a branch, and leave the actual work to an + // out-of-line function void destroy(value_t t) + { + if (JSON_HEDLEY_UNLIKELY(detail::owns_memory(t))) + { + destroy_owned(t); + } + } + + JSON_HEDLEY_NEVER_INLINE void destroy_owned(value_t t) { switch (t) { @@ -31507,11 +33552,16 @@ public: return next; } - /// @brief the capacity of the stored array, or unknown_size() - /// Only JSON_DIAGNOSTICS uses the value, to detect a reallocation that - /// would invalidate the parent pointers. Array types that do not have a - /// capacity() member function report unknown_size(), which is treated as - /// "the elements may have moved". + /// @brief the capacity of the stored array or object, or unknown_size() + /// Only JSON_DIAGNOSTICS uses the value, to detect an insertion that may + /// have moved the other elements and so invalidated their parent pointers. + /// unknown_size() means "the elements may have moved". Arrays only grow at + /// the end, so their capacity tells whether they reallocated; arrays + /// without a capacity() member report unknown_size(). For objects, only + /// types known to keep their elements in place (std::map and the other + /// node-based standard containers) report a fixed 0, and only + /// nlohmann::ordered_map, which appends members, reports its capacity; any + /// other object type (a sorted flat map, say) reports unknown_size(). #if JSON_DIAGNOSTICS template < typename A = array_t, detail::enable_if_t < detail::has_capacity::value, int > = 0 > std::size_t array_capacity() const noexcept @@ -31524,57 +33574,63 @@ public: { return detail::unknown_size(); } + + template < typename O = object_t, detail::enable_if_t < detail::has_stable_elements::value, int > = 0 > + static constexpr std::size_t object_capacity() noexcept + { + return 0; + } + + template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && detail::is_nlohmann_ordered_map::value, int > = 0 > + std::size_t object_capacity() const noexcept + { + return m_data.m_value.object->capacity(); + } + + template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && !detail::is_nlohmann_ordered_map::value, int > = 0 > + static constexpr std::size_t object_capacity() noexcept + { + return detail::unknown_size(); + } + + std::size_t container_capacity() const noexcept + { + return is_array() ? array_capacity() : object_capacity(); + } #else - static constexpr std::size_t array_capacity() noexcept + static constexpr std::size_t container_capacity() noexcept { return detail::unknown_size(); } #endif - /// @brief set the parent of a value that has just been added to an array - /// @param j the added value - /// @param old_capacity the value @ref array_capacity() returned before the + /// @brief set the parent of a value that has just been added to an array or object + /// @param j the added (or looked-up) value + /// @param old_capacity the value @ref container_capacity() returned before the /// insertion - reference set_parent_after_array_insert(reference j, std::size_t old_capacity) + reference set_parent_after_insert(reference j, std::size_t old_capacity) { #if JSON_DIAGNOSTICS // see https://github.com/nlohmann/json/issues/2838 - JSON_ASSERT(type() == value_t::array); if (JSON_HEDLEY_UNLIKELY(old_capacity == detail::unknown_size() - || array_capacity() != old_capacity)) + || container_capacity() != old_capacity)) { - // the capacity has changed, or the array type does not let us tell: - // the elements may have moved, so update all parents + // the capacity has changed, or the container type does not let us + // tell: the elements may have moved, so update all parents set_parents(); return j; } #else static_cast(old_capacity); #endif - return set_parent(j); + return set_parent_at(j); } - reference set_parent(reference j) + /// @brief set the parent of an element accessed via at(); never moves elements + reference set_parent_at(reference j) noexcept { #if JSON_DIAGNOSTICS - // ordered_json uses a vector internally, so pointers could have - // been invalidated; see https://github.com/nlohmann/json/issues/2962 -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - set_parents(); - return j; - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif - j.m_parent = this; -#else - static_cast(j); #endif return j; } @@ -32564,9 +34620,11 @@ public: /// @brief restore the parent pointers after erasing from an object - /// ordered_json keeps its members in a vector, and erasing a member - /// re-constructs every member after it in place, which resets their - /// parent pointers + /// Unless the object type is known to keep its elements in place (see + /// detail::has_stable_elements), erasing a member may move the members + /// after it; ordered_json, for instance, keeps its members in a vector + /// and re-constructs every member after the erased one in place, which + /// resets their parent pointers void set_parents_after_object_erase() { #if JSON_DIAGNOSTICS @@ -32574,7 +34632,7 @@ public: #pragma warning(push ) #pragma warning(disable : 4127) // ignore warning to replace if with if constexpr #endif - if (detail::is_ordered_map::value) + if (!detail::has_stable_elements::value) { set_parents(); } @@ -32609,7 +34667,7 @@ public: /// @brief create an empty value with a given type /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(const value_t v) - : m_data(v) + : m_data(stored_kind(v)) { assert_invariant(); } @@ -32690,7 +34748,7 @@ public: { convert_leaf(val); } - JSON_ASSERT(m_data.m_type == val.type()); + JSON_ASSERT(m_data.m_type == stored_kind(val.type())); set_parents(); assert_invariant(); @@ -32738,7 +34796,7 @@ public: // if an object is wanted but impossible, throw an exception if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object)) { - JSON_THROW(type_error::create(detail::exception_id::object_from_non_pairs, "cannot create object from initializer list", nullptr)); + detail::throw_literal(detail::exception_id::object_from_non_pairs, "cannot create object from initializer list", nullptr); } } @@ -32858,7 +34916,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterators_incompatible, "iterators are not compatible", nullptr)); + detail::throw_literal(detail::exception_id::iterators_incompatible, "iterators are not compatible", nullptr); } // copy type from the first iterator @@ -32877,7 +34935,7 @@ public: if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() || !last.m_it.primitive_iterator.is_end())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_out_of_range, "iterators out of range", first.m_object)); + detail::throw_literal(detail::exception_id::iterator_range_out_of_range, "iterators out of range", first.m_object); } break; } @@ -32945,7 +35003,7 @@ public: case value_t::null: case value_t::discarded: default: - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", first.m_object->type_name()), first.m_object)); + throw_iterator_range_of_null(first.m_object); } set_parents(); @@ -32974,18 +35032,16 @@ public: // check of passed value is valid other.assert_invariant(); - if (m_data.m_type == value_t::object || m_data.m_type == value_t::array) + if (JSON_HEDLEY_LIKELY(!detail::owns_memory(m_data.m_type))) { - // copying the container directly would call this constructor again - // for every element, once per nesting level - copy_structured(other); + // null, boolean, numbers, discarded: plain copy of the union + m_data.m_value = other.m_data.m_value; } else { - copy_leaf_value(other, *this); + copy_owning(other); } - set_parents(); assert_invariant(); } @@ -33215,6 +35271,22 @@ public: // value access // ////////////////// + /// out-of-line part of the copy constructor for owning types + JSON_HEDLEY_NEVER_INLINE + void copy_owning(const basic_json& other) + { + if (m_data.m_type == value_t::object || m_data.m_type == value_t::array) + { + // copying the container directly would call this constructor again + // for every element, once per nesting level + copy_structured(other); + } + else + { + copy_leaf_value(other, *this); + } + } + /// get a pointer to the value (object) object_t* get_impl_ptr(object_t* /*unused*/) noexcept { @@ -33275,14 +35347,16 @@ public: return m_data.m_type == value_t::number_integer ? &m_data.m_value.number_integer : nullptr; } - /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + /// get a pointer to the value (unsigned number); takes a placeholder if + /// number_unsigned_t is number_integer_t (the integer overload handles it then) + number_unsigned_t* get_impl_ptr(detail::number_unsigned_overload_type* /*unused*/) noexcept { return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } - /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + /// get a pointer to the value (unsigned number); takes a placeholder if + /// number_unsigned_t is number_integer_t (the integer overload handles it then) + constexpr const number_unsigned_t* get_impl_ptr(const detail::number_unsigned_overload_type* /*unused*/) const noexcept { return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } @@ -33311,6 +35385,40 @@ public: return is_binary() ? m_data.m_value.binary : nullptr; } + /// out-of-line throw helpers (message construction kept out of the hot paths) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_iterator_range_of_null(const basic_json* obj) + { + JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", obj->type_name()), obj)); + } + + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_incompatible_reference_type(const basic_json& obj) + { + JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); + } + + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_array_index_exceeds_size_type(size_type idx) + { + JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); + } + + /// @throw out_of_range.401 + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_array_index_out_of_range(size_type idx, const basic_json& j) + { + JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + } + + /// @throw out_of_range.403 + template + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_key_not_found(const KeyType& key, const basic_json& j) + { + JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key, "' not found"), &j)); + } + /*! @brief helper function to implement get_ref() @@ -33333,7 +35441,7 @@ public: return *ptr; } - JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); + throw_incompatible_reference_type(obj); } public: @@ -33781,7 +35889,7 @@ public: // std::map or ordered_map) never moves from its argument, so key is still // valid here regardless of whether KeyType was deduced as an rvalue reference // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved) - JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key_for_message(key), "' not found"), &j)); + throw_key_not_found(key_for_message(key), j); } return it->second; } @@ -33818,7 +35926,7 @@ public: if (JSON_HEDLEY_UNLIKELY(idx >= j.m_data.m_value.array->size())) { - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + throw_array_index_out_of_range(idx, j); } return (*j.m_data.m_value.array)[idx]; @@ -33873,7 +35981,7 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(size_type idx) { - return set_parent(array_at(*this, idx)); + return set_parent_at(array_at(*this, idx)); } /// @brief access specified array element with bounds checking @@ -33887,7 +35995,7 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(const typename object_t::key_type& key) { - return set_parent(object_at(*this, key)); + return set_parent_at(object_at(*this, key)); } /// @brief access specified object element with bounds checking @@ -33896,7 +36004,7 @@ public: detail::is_usable_as_basic_json_key_type::value, int> = 0> reference at(KeyType && key) { - return set_parent(object_at(*this, std::forward(key))); + return set_parent_at(object_at(*this, std::forward(key))); } /// @brief access specified object element with bounds checking @@ -33936,18 +36044,18 @@ public: // resize() rejects other indices that are too large to represent if (JSON_HEDLEY_UNLIKELY(idx == (std::numeric_limits::max)())) { - JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); + throw_array_index_exceeds_size_type(idx); } #if JSON_DIAGNOSTICS // remember array size & capacity before resizing const auto old_size = m_data.m_value.array->size(); - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); #endif m_data.m_value.array->resize(idx + 1); #if JSON_DIAGNOSTICS if (JSON_HEDLEY_UNLIKELY(old_capacity == detail::unknown_size() - || array_capacity() != old_capacity)) + || container_capacity() != old_capacity)) { // capacity has changed: update all parents set_parents(); @@ -33994,8 +36102,9 @@ public: // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { + const auto old_capacity = container_capacity(); auto result = m_data.m_value.object->emplace(std::move(key), nullptr); - return set_parent(result.first->second); + return set_parent_after_insert(result.first->second, old_capacity); } throw_subscript_wrong_type("string"); @@ -34045,8 +36154,9 @@ public: // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { + const auto old_capacity = container_capacity(); auto result = m_data.m_value.object->emplace(lookup_key(std::forward(key)), nullptr); - return set_parent(result.first->second); + return set_parent_after_insert(result.first->second, old_capacity); } throw_subscript_wrong_type("string"); @@ -34272,7 +36382,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(this != pos.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } IteratorType result = end(); @@ -34288,7 +36398,7 @@ public: { if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_out_of_range, "iterator out of range", this)); + detail::throw_literal(detail::exception_id::iterator_out_of_range, "iterator out of range", this); } m_data.m_value.destroy(m_data.m_type); @@ -34330,7 +36440,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_from_other_value, "iterators do not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_range_from_other_value, "iterators do not fit current value", this); } IteratorType result = end(); @@ -34347,7 +36457,7 @@ public: if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() || !last.m_it.primitive_iterator.is_end())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_out_of_range, "iterators out of range", this)); + detail::throw_literal(detail::exception_id::iterator_range_out_of_range, "iterators out of range", this); } m_data.m_value.destroy(m_data.m_type); @@ -34446,7 +36556,7 @@ public: { if (JSON_HEDLEY_UNLIKELY(idx >= size())) { - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); + throw_array_index_out_of_range(idx, *this); } m_data.m_value.array->erase(m_data.m_value.array->begin() + static_cast(idx)); @@ -34952,9 +37062,9 @@ public: } // add the element to the array (move semantics) - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->push_back(std::move(val)); - set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + set_parent_after_insert(m_data.m_value.array->back(), old_capacity); // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor } @@ -34983,9 +37093,9 @@ public: } // add the element to the array - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->push_back(val); - set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + set_parent_after_insert(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an array @@ -35013,8 +37123,9 @@ public: } // add the element to the object + const auto old_capacity = container_capacity(); auto res = m_data.m_value.object->insert(val); - set_parent(res.first->second); + set_parent_after_insert(res.first->second, old_capacity); } /// @brief add an object to an object @@ -35067,9 +37178,9 @@ public: } // add the element to the array (perfect forwarding) - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->emplace_back(std::forward(args)...); - return set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + return set_parent_after_insert(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an object if key does not exist @@ -35090,8 +37201,9 @@ public: } // add the element to the array (perfect forwarding) + const auto old_capacity = container_capacity(); auto res = m_data.m_value.object->emplace(std::forward(args)...); - set_parent(res.first->second); + set_parent_after_insert(res.first->second, old_capacity); // create a result iterator and set iterator to the result of emplace auto it = begin(); @@ -35132,7 +37244,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // insert to array and return iterator @@ -35152,7 +37264,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // moving into a local first keeps this safe even if val aliases @@ -35174,7 +37286,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // insert to array and return iterator @@ -35197,24 +37309,24 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } if (JSON_HEDLEY_UNLIKELY(first.m_object == this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_into_itself, "passed iterators may not belong to container", this)); + detail::throw_literal(detail::exception_id::insert_range_into_itself, "passed iterators may not belong to container", this); } // passed iterators must belong to arrays if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_array())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterators first and last must point to arrays", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to arrays", this); } // insert to array and return iterator @@ -35234,7 +37346,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // copy the values first: ilist may refer to elements of this array @@ -35262,13 +37374,13 @@ public: // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } // passed iterators must belong to objects if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this); } m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); @@ -35303,7 +37415,7 @@ public: // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } // passed iterators must belong to objects @@ -35390,9 +37502,9 @@ public: continue; } } - // set_parent() also repairs the other members, which ordered_json - // relocates when adding a key makes its vector grow - set_parent(m_data.m_value.object->operator[](it.key()) = std::move(it.value())); + // operator[] repairs the other members if the object type moves + // them; assigning through it keeps this O(1) per key + operator[](it.key()) = std::move(it.value()); } } @@ -35442,9 +37554,8 @@ public: continue; } } - // set_parent() also repairs the other members, which ordered_json - // relocates when adding a key makes its vector grow - target->set_parent(target->m_data.m_value.object->operator[](first.key()) = std::move(first.value())); + // operator[] repairs the other members if the object type moves them + target->operator[](first.key()) = std::move(first.value()); ++first; } } @@ -35586,7 +37697,22 @@ public: /// @name lexicographical comparison operators /// @{ - // note parentheses around operands are necessary; see + // the same-type cases for non-owning scalars, shared by JSON_IMPLEMENT_OPERATOR + // and JSON_IMPLEMENT_SCALAR_FAST +#define JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + case value_t::null: \ + return (null_result); \ + case value_t::boolean: \ + return lhs.m_data.m_value.boolean op rhs.m_data.m_value.boolean; \ + case value_t::number_integer: \ + return lhs.m_data.m_value.number_integer op rhs.m_data.m_value.number_integer; \ + case value_t::number_unsigned: \ + return lhs.m_data.m_value.number_unsigned op rhs.m_data.m_value.number_unsigned; \ + case value_t::number_float: \ + return lhs.m_data.m_value.number_float op rhs.m_data.m_value.number_float; + + // note parentheses around the container operands are necessary, because + // GCC otherwise parses `array <` as std::array; see // https://github.com/nlohmann/json/issues/1530 // Mixed signed/unsigned integer comparisons check whether the signed value // is negative before casting. If it is, the comparison is performed with @@ -35622,24 +37748,11 @@ public: return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ } \ \ - case value_t::null: \ - return (null_result); \ - \ + JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + \ case value_t::string: \ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ - case value_t::boolean: \ - return (lhs.m_data.m_value.boolean) op (rhs.m_data.m_value.boolean); \ - \ - case value_t::number_integer: \ - return (lhs.m_data.m_value.number_integer) op (rhs.m_data.m_value.number_integer); \ - \ - case value_t::number_unsigned: \ - return (lhs.m_data.m_value.number_unsigned) op (rhs.m_data.m_value.number_unsigned); \ - \ - case value_t::number_float: \ - return (lhs.m_data.m_value.number_float) op (rhs.m_data.m_value.number_float); \ - \ case value_t::binary: \ return (*lhs.m_data.m_value.binary) op (*rhs.m_data.m_value.binary); \ \ @@ -35683,6 +37796,24 @@ public: \ return (default_result); + // inline fast path for two values of the same non-owning scalar type; anything + // else falls through to the out-of-line general comparison +#define JSON_IMPLEMENT_SCALAR_FAST(op, null_result) \ + if (lhs.m_data.m_type == rhs.m_data.m_type) \ + { \ + switch (lhs.m_data.m_type) \ + { \ + JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + case value_t::object: \ + case value_t::array: \ + case value_t::string: \ + case value_t::binary: \ + case value_t::discarded: \ + default: \ + break; \ + } \ + } + JSON_PRIVATE_UNLESS_TESTED: // returns true if: // - any operand is NaN and the other operand is of number type @@ -35710,6 +37841,173 @@ public: return compares_unordered(*this, rhs, inverse); } + /// @brief whether objects and arrays are the standard containers, whose + /// elements equal_bounded() may therefore compare itself + using standard_object_t = std::map>>; + using standard_array_t = std::vector>; + using equal_bounded_containers = detail::conjunction, std::is_same>; + + // out-of-line general comparisons; the operators below handle same-type + // scalars inline and fall through to these + JSON_HEDLEY_NEVER_INLINE + static bool equal_general(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + JSON_IMPLEMENT_OPERATOR( ==, true, false, false, + compare_iteratively(lhs, rhs, false) == compare_result::equal, true) +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + } + + JSON_HEDLEY_NEVER_INLINE + static bool less_general(const_reference lhs, const_reference rhs) noexcept + { + // default_result is used if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + compare_iteratively(lhs, rhs, true) == compare_result::less, false) + } + +#if JSON_HAS_THREE_WAY_COMPARISON + JSON_HEDLEY_NEVER_INLINE + static std::partial_ordering three_way_general(const_reference lhs, const_reference rhs) noexcept // *NOPAD* + { + // default_result is used if we cannot compare values. In that case, + // we compare types. + JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* + std::partial_ordering::equivalent, + std::partial_ordering::unordered, + lhs_type <=> rhs_type, // *NOPAD* + to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) + } +#endif + + /// @brief compare two leaves of the same type inside equal_structured() + /// @return 1 if they are equal, 0 if they are not, and -1 if the values are + /// of different types or are not handled here, which this leaves to + /// equal_structured() + /// @note Unlike JSON_IMPLEMENT_SCALAR_FAST, this also compares strings: + /// they are the most common leaves in a document, so comparing them + /// inline pays off in the container walk, while the operators keep + /// their inlined code at each call site small. Binary values are + /// rare and left to equal_general(). + JSON_HEDLEY_ALWAYS_INLINE + static int equal_leaf(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + if (lhs.m_data.m_type == rhs.m_data.m_type) + { + switch (lhs.m_data.m_type) + { + case value_t::null: + return 1; + case value_t::boolean: + return lhs.m_data.m_value.boolean == rhs.m_data.m_value.boolean; + case value_t::number_integer: + return lhs.m_data.m_value.number_integer == rhs.m_data.m_value.number_integer; + case value_t::number_unsigned: + return lhs.m_data.m_value.number_unsigned == rhs.m_data.m_value.number_unsigned; + case value_t::number_float: + return lhs.m_data.m_value.number_float == rhs.m_data.m_value.number_float; + case value_t::string: + return *lhs.m_data.m_value.string == *rhs.m_data.m_value.string; + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + } +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + return -1; + } + + /// @brief compare for equality, walking objects and arrays here instead of + /// in their containers so that the depth is a parameter and needs no + /// thread-local count; only used for the standard containers + static bool equal_bounded(const_reference lhs, const_reference rhs, std::size_t depth) noexcept + { + const int leaf = equal_leaf(lhs, rhs); + return leaf < 0 ? equal_structured(lhs, rhs, depth) : leaf != 0; + } + + static bool equal_structured(const_reference lhs, const_reference rhs, std::size_t depth) noexcept + { + const auto type = lhs.m_data.m_type; + if (type != rhs.m_data.m_type || (type != value_t::array && type != value_t::object)) + { + return equal_general(lhs, rhs); + } + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + return compare_iteratively(lhs, rhs, false) == compare_result::equal; + } + + if (type == value_t::array) + { + const auto& a = *lhs.m_data.m_value.array; + const auto& b = *rhs.m_data.m_value.array; + if (a.size() != b.size()) + { + return false; + } + auto it_b = b.cbegin(); + for (auto it_a = a.cbegin(); it_a != a.cend(); ++it_a, ++it_b) + { + const int leaf = equal_leaf(*it_a, *it_b); + if (leaf < 0 ? !equal_structured(*it_a, *it_b, depth + 1) : leaf == 0) + { + return false; + } + } + return true; + } + + const auto& a = *lhs.m_data.m_value.object; + const auto& b = *rhs.m_data.m_value.object; + if (a.size() != b.size()) + { + return false; + } + auto it_b = b.cbegin(); + for (auto it_a = a.cbegin(); it_a != a.cend(); ++it_a, ++it_b) + { + if (!(it_a->first == it_b->first)) + { + return false; + } + const int leaf = equal_leaf(it_a->second, it_b->second); + if (leaf < 0 ? !equal_structured(it_a->second, it_b->second, depth + 1) : leaf == 0) + { + return false; + } + } + return true; + } + + static bool equal_dispatch(const_reference lhs, const_reference rhs, std::true_type /*standard containers*/) noexcept + { + return equal_bounded(lhs, rhs, 0); + } + + static bool equal_dispatch(const_reference lhs, const_reference rhs, std::false_type /*other containers*/) noexcept + { + return equal_general(lhs, rhs); + } + public: #if JSON_HAS_THREE_WAY_COMPARISON /// @brief comparison: equal @@ -35721,11 +38019,11 @@ public: JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif const_reference lhs = *this; - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, - compare_iteratively(lhs, rhs, false) == compare_result::equal, true) + JSON_IMPLEMENT_SCALAR_FAST( ==, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP #endif + return equal_dispatch(lhs, rhs, equal_bounded_containers {}); } /// @brief comparison: equal @@ -35742,13 +38040,8 @@ public: std::partial_ordering operator<=>(const_reference rhs) const noexcept // *NOPAD* { const_reference lhs = *this; - // default_result is used if we cannot compare values. In that case, - // we compare types. - JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* - std::partial_ordering::equivalent, - std::partial_ordering::unordered, - lhs_type <=> rhs_type, // *NOPAD* - to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) + JSON_IMPLEMENT_SCALAR_FAST(<=>, std::partial_ordering::equivalent) // *NOPAD* + return three_way_general(lhs, rhs); } /// @brief comparison: 3-way @@ -35836,11 +38129,11 @@ public: JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, - compare_iteratively(lhs, rhs, false) == compare_result::equal, true) + JSON_IMPLEMENT_SCALAR_FAST( ==, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP #endif + return equal_dispatch(lhs, rhs, equal_bounded_containers {}); } // The friend comparisons with a scalar name the JSON type via decltype of @@ -35895,11 +38188,8 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ friend bool operator<(const_reference lhs, const_reference rhs) noexcept { - // default_result is used if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), - compare_iteratively(lhs, rhs, true) == compare_result::less, false) + JSON_IMPLEMENT_SCALAR_FAST( <, false) + return less_general(lhs, rhs); } /// @brief comparison: less than @@ -36010,6 +38300,8 @@ public: #endif #undef JSON_IMPLEMENT_OPERATOR +#undef JSON_IMPLEMENT_SCALAR_FAST +#undef JSON_IMPLEMENT_SCALAR_CASES /// @} @@ -36990,24 +39282,18 @@ public: }; // wrapper for "add" operation; add value at ptr - const auto operation_add = [&result](json_pointer & ptr, const basic_json & val) + const auto operation_add = [&result](json_pointer & ptr, basic_json val) { // adding to the root of the target document means replacing it if (ptr.empty()) { - result = val; + result = std::move(val); return; } - // make sure the top element of the pointer exists - json_pointer const top_pointer = ptr.top(); - if (top_pointer != ptr) - { - result.at(top_pointer); - } - - // get reference to the parent of the JSON pointer ptr - const auto last_path = ptr.back(); + // get reference to the parent of the JSON pointer ptr; a missing + // parent (or one of its ancestors) is reported by at() + auto last_path = std::move(ptr.reference_tokens.back()); ptr.pop_back(); // parent must exist when performing patch add per RFC6902 specs basic_json& parent = result.at(ptr); @@ -37018,7 +39304,7 @@ public: case value_t::object: { // use operator[] to add value - parent[last_path] = val; + parent[std::move(last_path)] = std::move(val); break; } @@ -37027,7 +39313,7 @@ public: if (last_path == "-") { // special case: append to back - parent.push_back(val); + parent.push_back(std::move(val)); } else { @@ -37035,11 +39321,11 @@ public: if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) { // avoid undefined behavior - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &parent)); + throw_array_index_out_of_range(idx, parent); } // default case: insert add offset - parent.insert(parent.begin() + static_cast(idx), val); + parent.insert(parent.begin() + static_cast(idx), std::move(val)); } break; } @@ -37054,7 +39340,7 @@ public: case value_t::binary: case value_t::discarded: default: - JSON_THROW(out_of_range::create(detail::exception_id::patch_add_parent_not_container, detail::concat("cannot add value: the JSON Patch 'add' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + detail::throw_patch_add_parent(parent); } }; @@ -37062,7 +39348,8 @@ public: const auto operation_remove = [this, & result](json_pointer & ptr) { // get reference to the parent of the JSON pointer ptr - const auto last_path = ptr.back(); + static_cast(ptr.back()); // throws out_of_range.405 if ptr is the root + const auto last_path = std::move(ptr.reference_tokens.back()); ptr.pop_back(); basic_json& parent = result.at(ptr); @@ -37077,7 +39364,7 @@ public: } else { - JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", last_path, "' not found"), this)); + throw_key_not_found(last_path, *this); } } else if (parent.is_array()) @@ -37089,7 +39376,7 @@ public: { // the parent of a "remove" target must be an object or array // (see #5396) - JSON_THROW(out_of_range::create(detail::exception_id::patch_remove_parent_not_container, detail::concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + detail::throw_patch_remove_parent(parent); } }; @@ -37120,7 +39407,7 @@ public: // type check: top level value must be an array if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array())) { - JSON_THROW(parse_error::create(detail::exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &json_patch)); + detail::throw_patch_not_an_array(json_patch); } // iterate and apply the operations @@ -37134,21 +39421,16 @@ public: // find value auto it = val.m_data.m_value.object->find(member); - // context-sensitive error message - const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\''); // NOLINT(bugprone-unused-local-non-trivial-variable) - // check if the desired value is present if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end())) { - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat(error_msg, " must have member '", member, "'"), &val)); + detail::throw_patch_missing_member(op, member, false, val); } // check if the result is of type string if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) { - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat(error_msg, " must have string member '", member, "'"), &val)); + detail::throw_patch_missing_member(op, member, true, val); } // no error: return value @@ -37158,12 +39440,12 @@ public: // type check: every element of the array must be an object if (JSON_HEDLEY_UNLIKELY(!val.is_object())) { - JSON_THROW(parse_error::create(detail::exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &val)); + detail::throw_patch_not_an_array(val); } // collect mandatory members - const auto op = get_value("op", "op", true).template get(); - const auto path = get_value(op, "path", true).template get(); + const string_t& op = *get_value("op", "op", true).m_data.m_value.string; + const string_t& path = *get_value(op, "path", true).m_data.m_value.string; json_pointer ptr(path); switch (get_op(op)) @@ -37189,38 +39471,37 @@ public: case patch_operations::move: { - const auto from_path = get_value("move", "from", true).template get(); + const string_t& from_path = *get_value("move", "from", true).m_data.m_value.string; json_pointer from_ptr(from_path); if (JSON_HEDLEY_UNLIKELY(is_proper_prefix(from_ptr, ptr))) { - JSON_THROW(out_of_range::create(detail::exception_id::patch_move_into_child, detail::concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); + detail::throw_patch_move_into_child(from_path, path, result); } // the "from" location must exist - use at() - basic_json const v = result.at(from_ptr); + // a root "from" is rejected by operation_remove below, which must leave result intact + basic_json v = from_ptr.empty() ? basic_json(result) : std::move(result.at(from_ptr)); // The move operation is functionally identical to a // "remove" operation on the "from" location, followed // immediately by an "add" operation at the target // location with the value that was just removed. operation_remove(from_ptr); - operation_add(ptr, v); + operation_add(ptr, std::move(v)); break; } case patch_operations::copy: { - const auto from_path = get_value("copy", "from", true).template get(); + const string_t& from_path = *get_value("copy", "from", true).m_data.m_value.string; const json_pointer from_ptr(from_path); - // the "from" location must exist - use at() - basic_json const v = result.at(from_ptr); - // The copy is functionally identical to an "add" // operation at the target location using the value - // specified in the "from" member. - operation_add(ptr, v); + // specified in the "from" member; the "from" location + // must exist - use at() + operation_add(ptr, result.at(from_ptr)); break; } @@ -37241,7 +39522,7 @@ public: // throw an exception if the test fails if (JSON_HEDLEY_UNLIKELY(!success)) { - JSON_THROW(other_error::create(detail::exception_id::patch_test_failed, detail::concat("unsuccessful: ", val.dump()), &val)); + detail::throw_patch_test_failed(val); } break; @@ -37252,7 +39533,7 @@ public: { // op must be "add", "remove", "replace", "move", "copy", or // "test" - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat("operation value '", op, "' is invalid"), &val)); + detail::throw_patch_invalid_operation(op, val); } } } @@ -37306,12 +39587,11 @@ public: std::size_t index = 0; /// objects: the next member of source to look at const_iterator member{}; // NOLINT(readability-redundant-member-init) - /// objects: the keys common to both, in source's order - std::vector common_keys{}; // NOLINT(readability-redundant-member-init) - /// objects: the next entry of common_keys - std::size_t next_common = 0; - /// objects: the "add" operations for keys only target has - basic_json added_ops{}; // NOLINT(readability-redundant-member-init) + /// objects: for each member of source (in order), its value in + /// target, or nullptr if target has no such key; `index` is the next + std::vector target_values{}; // NOLINT(readability-redundant-member-init) + /// objects: the members only target has, in target's order + std::vector added{}; // NOLINT(readability-redundant-member-init) }; // The operations of a diff are built by the functions below rather than @@ -37367,111 +39647,113 @@ public: } } + /// @brief appends the "add" operations for the members @a added to @a result + static void diff_added(basic_json& result, const std::vector& added, const string_t& path) + { + for (const auto& it : added) + { + diff_add(result, detail::concat(path, '/', detail::escape(it.key())), it.value()); + } + } + /*! @brief compare the keys of objects @a source and @a target If object_t does not keep its members in insertion order, or if the keys both objects have are in the same order in both, and the keys only - @a target has come after them, stores the keys common to both in - source's order in @a common_keys, stores the "add" operations for the keys - only @a target has in @a added_ops, and returns true: the caller then diffs - the objects member by member. Otherwise, appends operations that remove - every member of @a source and add every member of @a target to @a result, - and returns false. + @a target has come after them, stores in @a target_values, for each member + of source in order, its value in target (nullptr for a key target does not + have), stores the members only target has in @a added, and returns true: + the caller then diffs the objects member by member. Otherwise, appends + operations that remove every member of @a source and add every member of + @a target to @a result, and returns false. */ static bool diff_object_keys(basic_json& result, const basic_json& source, const basic_json& target, - const string_t& path, std::vector& common_keys, - basic_json& added_ops) + const string_t& path, std::vector& target_values, + std::vector& added) { - // first pass: record, for every source key, whether it is - // common to both objects (in source's iteration order) or - // was deleted (i.e., in source but not in target) -- this is - // a by-product of the target.find() call already needed to - // tell the two cases apart, so it adds no extra lookups. The - // "remove" ops themselves are emitted later, interleaved - // with the per-key diffs in the caller's fast path, to match - // source's original iteration order (as the original, - // pre-reordering-aware implementation did) instead of - // grouping all removes before all per-key diffs. - std::vector common_keys_source_order; + // first pass: one lookup per source key + target_values.reserve(source.size()); + std::size_t common = 0; for (auto it = source.cbegin(); it != source.cend(); ++it) { - if (target.find(it.key()) != target.end()) + const auto found = target.find(it.key()); + if (found != target.cend()) { - common_keys_source_order.push_back(it.key()); - } - } - - // second pass: find keys that were added (i.e., in target but - // not in source), and record the keys common to both, in - // target's iteration order -- again a by-product of the - // source.find() call already needed to detect added keys. At - // the same time, determine whether every added key comes - // after every common key in target's order (a precondition - // for the fast path, which only ever appends new keys - // at the very end). Both are only needed for an object_t that - // keeps its members in insertion order, such as the one - // backing `ordered_json`; for any other object_t, the fast - // path is always taken and they are not computed. - // The patch ops for keys that were added (i.e., in target but not - // in source) are built here so the fast path can reuse - // them without a second source.find() per target key. Only - // used by the fast path -- the slow (reordering) path - // rebuilds "add" ops for every key itself. - std::vector common_keys_target_order; - bool new_keys_form_suffix = true; - bool seen_new_key = false; - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - if (source.find(it.key()) == source.end()) - { - seen_new_key = true; - diff_add(added_ops, detail::concat(path, '/', detail::escape(it.key())), it.value()); + target_values.push_back(&found.value()); + ++common; } else { -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - common_keys_target_order.push_back(it.key()); - if (seen_new_key) - { - new_keys_form_suffix = false; - } - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif + target_values.push_back(nullptr); } } - // Only an object type that keeps its members in insertion - // order, such as nlohmann::ordered_map, can need reordering: - // patch() appends a new member at the end of such an object. - // Any other object type places its members itself - std::map - // in key order, a hash map in an order its operator== ignores - - // so a member-by-member diff always reproduces target there. + // second pass: find the keys only target has. An object type that + // keeps its members in insertion order, such as the one backing + // ordered_json, also needs the common keys to be in the same order + // in both objects, and the new keys to come after them, as patch() + // appends a new member at the end of such an object. Any other + // object type places its members itself, so a member-by-member diff + // always reproduces target there. #ifdef JSON_HEDLEY_MSVC_VERSION #pragma warning(push ) #pragma warning(disable : 4127) // ignore warning to replace if with if constexpr #endif - if (!detail::is_ordered_map::value - || (common_keys_source_order == common_keys_target_order && new_keys_form_suffix)) + bool fast_path = true; + if (detail::is_ordered_map::value) { - // fast path: order of common keys already matches (or the - // object_t's iteration order does not depend on - // insertion history), so a plain per-key diff is correct - // and minimal, as before - common_keys = std::move(common_keys_source_order); - return true; + // walk the common keys of source in lockstep with target: a key + // of target that is not the next one is either new or out of order + auto src = source.cbegin(); + std::size_t next = 0; + bool seen_new = false; + for (auto it = target.cbegin(); it != target.cend() && fast_path; ++it) + { + while (next < target_values.size() && target_values[next] == nullptr) + { + ++src; + ++next; + } + + if (next < target_values.size() && src.key() == it.key()) + { + // a common key after a new one: new keys are no suffix + fast_path = !seen_new; + ++src; + ++next; + } + else if (source.find(it.key()) == source.cend()) + { + seen_new = true; + added.push_back(it); + } + else + { + // a common key out of order + fast_path = false; + } + } + } + else if (target.size() != common) + { + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.cend()) + { + added.push_back(it); + } + } } #ifdef JSON_HEDLEY_MSVC_VERSION #pragma warning( pop ) #endif + if (fast_path) + { + return true; + } + // slow path: the common keys are in a different relative // order in source and target (only possible for a // reorderable object_t like ordered_map). Building a @@ -37521,6 +39803,14 @@ public: return; } + diff_unequal(result, source, target, path, depth); + } + + /// @brief @ref diff_recursively for values known to be unequal; callers + /// compare first so they build no path for equal ones + static void diff_unequal(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, const std::size_t depth) + { if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) { diff_iteratively(result, source, target, path); @@ -37543,7 +39833,10 @@ public: while (i < source.size() && i < target.size()) { // recursive call to compare array values at index i - diff_recursively(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); + if (!(source[i] == target[i])) + { + diff_unequal(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); + } ++i; } @@ -37555,25 +39848,24 @@ public: case value_t::object: { - std::vector common_keys; - basic_json added_ops(value_t::array); - if (diff_object_keys(result, source, target, path, common_keys, added_ops)) + std::vector target_values; + std::vector added; + if (diff_object_keys(result, source, target, path, target_values, added)) { - // fast path: common_keys is, by construction, the - // subsequence of source's keys that are common to both - // objects, in source's iteration order -- so it can be - // walked in lockstep with `source` using a cheap key - // comparison instead of another lookup. Deleted keys - // (those source keys not in common_keys) are interleaved - // here too, in source's original order, to match the + // fast path: target_values holds, for every member of + // source in order, its value in target -- so no further + // lookup is needed. Deleted keys are interleaved here + // too, in source's original order, to match the // historical (pre-reordering-aware) output order. - auto common_it = common_keys.cbegin(); - for (auto it = source.cbegin(); it != source.cend(); ++it) + std::size_t i = 0; + for (auto it = source.cbegin(); it != source.cend(); ++it, ++i) { - if (common_it != common_keys.cend() && it.key() == *common_it) + if (target_values[i] != nullptr) { - diff_recursively(result, it.value(), target[it.key()], detail::concat(path, '/', detail::escape(it.key())), depth + 1); - ++common_it; + if (!(it.value() == *target_values[i])) + { + diff_unequal(result, it.value(), *target_values[i], detail::concat(path, '/', detail::escape(it.key())), depth + 1); + } } else { @@ -37582,10 +39874,8 @@ public: } } - // append the "add" ops for brand-new keys collected by - // diff_object_keys -- no second source.find() per target - // key needed - result.insert(result.end(), added_ops.begin(), added_ops.end()); + // append the "add" operations for brand-new keys + diff_added(result, added, path); } break; } @@ -37658,17 +39948,17 @@ public: case value_t::object: { - std::vector common_keys; - basic_json added_ops(value_t::array); - if (diff_object_keys(result, s, t, current_path, common_keys, added_ops)) + std::vector target_values; + std::vector added; + if (diff_object_keys(result, s, t, current_path, target_values, added)) { // fast path: the frame walks source in lockstep with // common_keys, as diff_recursively does, and appends // added_ops once all members are done stack.emplace_back(&s, &t, current_path.size()); stack.back().member = s.cbegin(); - stack.back().common_keys = std::move(common_keys); - stack.back().added_ops = std::move(added_ops); + stack.back().target_values = std::move(target_values); + stack.back().added = std::move(added); } return; } @@ -37730,11 +40020,10 @@ public: if (it != s->cend()) { ++stack.back().member; - const std::size_t next_common = stack.back().next_common; - if (next_common < stack.back().common_keys.size() && it.key() == stack.back().common_keys[next_common]) + const basic_json* const target_value_ptr = stack.back().target_values[stack.back().index++]; + if (target_value_ptr != nullptr) { - ++stack.back().next_common; - const basic_json& target_value = (*t)[it.key()]; + const basic_json& target_value = *target_value_ptr; detail::concat_into(current_path, '/', detail::escape(it.key())); enter(it.value(), target_value); if (stack.size() == depth) @@ -37752,7 +40041,7 @@ public: // append the "add" ops for brand-new keys collected when the // object was entered - result.insert(result.end(), stack.back().added_ops.begin(), stack.back().added_ops.end()); + diff_added(result, stack.back().added, current_path); } // this array or object is done: continue with the one it is in @@ -37882,6 +40171,176 @@ public: } public: + /// @brief creates a diff as a JSON Merge Patch + /// @sa https://json.nlohmann.me/api/basic_json/merge_diff/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json merge_diff(const basic_json& source, const basic_json& target) + { + // unless both are objects, the patch is the target itself + if (!source.is_object() || !target.is_object()) + { + return target; + } + + return merge_diff_recursively(source, target, 0); + } + + private: + /// @brief two objects @ref merge_diff_iteratively is diffing + struct merge_diff_frame + { + merge_diff_frame(const basic_json* source_, const basic_json* target_) + : source(source_), target(target_), member(source_->cbegin()), patch(value_t::object) + {} + + // declared for GCC's -Weffc++, as in diff_frame + merge_diff_frame(const merge_diff_frame&) = default; + merge_diff_frame(merge_diff_frame&&) = default; + merge_diff_frame& operator=(const merge_diff_frame&) = default; + merge_diff_frame& operator=(merge_diff_frame&&) = default; + ~merge_diff_frame() = default; + + /// the objects being diffed + const basic_json* source; + const basic_json* target; + /// the member of source being diffed; advanced once it is done + const_iterator member; + /// the patch for the members of source diffed so far + basic_json patch; + }; + + /// @brief add the members of object @a target that @a source does not + /// have to the merge patch @a result + static void merge_diff_added_members(basic_json& result, const basic_json& source, const basic_json& target) + { + for (auto it = target.begin(); it != target.end(); ++it) + { + auto itf = source.find(it.key()); + if (itf == source.end()) + { + result[it.key()] = it.value(); + } + } + } + + /*! + @brief @ref merge_diff of two objects at nesting level @a depth + + Diffing two nested objects calls this function again, once per nesting + level, so values nested deeply enough would exhaust the call stack and + terminate the process. The descent is bounded here: once @ref + detail::recursion_depth_limit levels have been entered, @ref + merge_diff_iteratively diffs what is left without the call stack. + */ + static basic_json merge_diff_recursively(const basic_json& source, const basic_json& target, const std::size_t depth) + { + JSON_ASSERT(source.is_object() && target.is_object()); + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + return merge_diff_iteratively(source, target); + } + + basic_json result(value_t::object); + + for (auto it = source.begin(); it != source.end(); ++it) + { + auto itf = target.find(it.key()); + if (itf != target.end()) + { + if (it.value().is_object() && itf.value().is_object()) + { + auto diff = merge_diff_recursively(it.value(), itf.value(), depth + 1); + if (!diff.empty()) + { + result[it.key()] = std::move(diff); + } + } + else if (it.value() != itf.value()) + { + result[it.key()] = itf.value(); + } + } + else + { + result[it.key()] = value_t::null; + } + } + + merge_diff_added_members(result, source, target); + return result; + } + + /*! + @brief @ref merge_diff of two objects without the call stack + + Produces the same patch as @ref merge_diff_recursively, keeping the objects + being diffed on an explicit stack instead of descending into them. Two + nested objects are still diffed completely before the next member, so the + patch has its members in the same order as in the recursive version. Only + reached for objects nested more deeply than @ref + detail::recursion_depth_limit. + */ + static basic_json merge_diff_iteratively(const basic_json& source, const basic_json& target) + { + // the patch of source and target, set once their frame is popped + basic_json result; + std::vector stack; + stack.emplace_back(&source, &target); + while (!stack.empty()) + { + // the frame's members are copied out and it is changed through + // stack.back(): pushing a frame for nested objects or popping + // this one would invalidate a reference to it + const basic_json* const s = stack.back().source; + const basic_json* const t = stack.back().target; + const const_iterator it = stack.back().member; + + if (it != s->cend()) + { + const const_iterator itf = t->find(it.key()); + if (itf != t->cend()) + { + if (it.value().is_object() && itf.value().is_object()) + { + // member is advanced once their patch is under its key + stack.emplace_back(&it.value(), &itf.value()); + continue; + } + if (it.value() != itf.value()) + { + stack.back().patch[it.key()] = itf.value(); + } + } + else + { + stack.back().patch[it.key()] = value_t::null; + } + ++stack.back().member; + continue; + } + + merge_diff_added_members(stack.back().patch, *s, *t); + + // these objects are done: continue with the ones they are in + basic_json patch = std::move(stack.back().patch); + stack.pop_back(); + if (stack.empty()) + { + result = std::move(patch); + } + else + { + if (!patch.empty()) + { + stack.back().patch[stack.back().member.key()] = std::move(patch); + } + ++stack.back().member; + } + } + return result; + } + /// @} }; @@ -38075,6 +40534,8 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_INTERNAL_ALWAYS_INLINE #undef JSON_DISABLE_ENUM_SERIALIZATION #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION +#undef JSON_DTOA_SSE2 +#undef JSON_DTOA_NEON #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH @@ -38352,7 +40813,7 @@ NLOHMANN_JSON_NAMESPACE_END #undef NLOHMANN_JSON_LITERAL_OPERATOR #endif // INCLUDE_NLOHMANN_JSON_LITERALS_HPP_ - +// IWYU pragma: export #endif #endif // INCLUDE_NLOHMANN_JSON_HPP_ diff --git a/tests/benchmarks/CMakeLists.txt b/tests/benchmarks/CMakeLists.txt index 4d7265145..e4ae29923 100644 --- a/tests/benchmarks/CMakeLists.txt +++ b/tests/benchmarks/CMakeLists.txt @@ -6,8 +6,9 @@ if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang)) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -DNDEBUG -O3") endif() -# configure Google Benchmark; a fixed release, so that results stay comparable -set(JSON_GOOGLE_BENCHMARK_VERSION 1.9.5) +# configure Google Benchmark; a fixed release with a checksum, so that builds are reproducible. Update it like any +# other dependency; comparisons stay fair because both sides are built with the same version (see README.md) +set(JSON_GOOGLE_BENCHMARK_VERSION 1.10.0) include(FetchContent) # only the library is needed; -Werror would break the pinned release as soon as @@ -18,7 +19,7 @@ set(BENCHMARK_ENABLE_WERROR OFF CACHE BOOL "" FORCE) FetchContent_Declare(benchmark URL https://github.com/google/benchmark/archive/refs/tags/v${JSON_GOOGLE_BENCHMARK_VERSION}.tar.gz - URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340 + URL_HASH SHA256=acc5c7c0fbed855c4aaa2d7c3c36fb82b0c15c28f4e151e3f679c2fc0aa404af DOWNLOAD_EXTRACT_TIMESTAMP TRUE ) FetchContent_MakeAvailable(benchmark) diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index aaf5abe46..6f067db60 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -27,8 +27,11 @@ The input files are those of [nativejson-benchmark](https://github.com/miloyip/n - Network access on the first configure: CMake downloads Google Benchmark and the [test data](https://github.com/nlohmann/json_test_data) into the build directory. To reuse a download of the test data, pass `-DJSON_TestDataDirectory=/test_files`. -- Google Benchmark is pinned to a release (1.9.5), so that results from different days stay comparable. To update it, - change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together. +- Google Benchmark is fetched as a fixed release with a checksum, so that every build uses the same + code and a download cannot change under you. It is updated like any other dependency when a new release comes out: + change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together. Results are kept + comparable by building both sides of a comparison with the same version, not by keeping that version forever (see + [Comparing two versions](#comparing-two-versions)). - The benchmarks include `single_include/nlohmann/json.hpp`, so run `make amalgamate` after changing anything in `include/`. @@ -110,13 +113,18 @@ Python, unpinned versions work as well. In its output: - `OVERALL_GEOMEAN` summarizes all benchmarks; - `-a` shows only the aggregates, not every repetition. +Both builds use the same Google Benchmark, the one `CMakeLists.txt` names, so only the header differs. For the same +reason, do not compare against result files from an earlier checkout: a different Google Benchmark, compiler or +machine state can shift the numbers as much as the change under test. Rerun the baseline instead. + The header you compare with must support everything the benchmarks use. The current benchmarks build against 3.12.0. Only benchmarks present in both result files are compared, so for older releases, either filter the benchmarks or build that release's own `tests/benchmarks` against its own header. ## Getting stable numbers -- Build and run both versions on the same machine, one right after the other. +- Build and run both versions on the same machine with the same compiler and Google Benchmark, one right after the + other. - Keep the machine otherwise idle: no builds, no browser, and a laptop plugged in. - On Linux, set the CPU frequency governor to `performance`, e.g. `sudo cpupower frequency-set --governor performance`. Google Benchmark prints a warning when frequency scaling is enabled. Pinning the process to a core diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index 5df7dd473..edc4f6088 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -9,7 +9,10 @@ #include #include #include +#include +#include #include +#include #include #include @@ -621,4 +624,27 @@ static void FromCborChunkedString(benchmark::State& state, const std::size_t chu BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000); +// conversion of a JSON array into standard containers +template +static void GetArray(benchmark::State& state) +{ + json j = json::array(); + for (int i = 0; i < 1000; ++i) + { + j.push_back(i); + } + + for (auto _ : state) + { + auto c = j.get(); + benchmark::DoNotOptimize(c); + } + state.SetItemsProcessed(state.iterations() * 1000); +} + +BENCHMARK_TEMPLATE(GetArray, std::vector); +BENCHMARK_TEMPLATE(GetArray, std::deque); +BENCHMARK_TEMPLATE(GetArray, std::set); +BENCHMARK_TEMPLATE(GetArray, std::array); + BENCHMARK_MAIN(); diff --git a/tests/fuzzing.md b/tests/fuzzing.md index 329cdbf58..8a1a169ea 100644 --- a/tests/fuzzing.md +++ b/tests/fuzzing.md @@ -9,7 +9,16 @@ Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once once with exceptions. Both calls must agree. Where parsing with exceptions fails, the call without exceptions must return a discarded value (or throw the same kind of non-parse error), and it must never throw a `parse_error`. Where parsing succeeds, both calls must return the same value. The drivers then serialize the value, parse the result back, -and check that nothing was lost. The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`. +and check that nothing was lost. + +Finally, each driver reads the same input through the other input adapters: a `std::istringstream`, a non-contiguous +iterator range (`std::list`), and, on POSIX systems, a `FILE*` from `fmemopen`. The pointer range takes the +fast paths for contiguous input (bulk string scanning, multi-byte loads, lazy position tracking), while the other +adapters read byte by byte. Every adapter must give the same value, or the same exception with the same id, message, +and byte position. The JSON driver checks this for `parse` and `accept`, each with and without `ignore_comments`. The +shared code is `check_input_adapters()` in `tests/src/fuzzer_common.hpp`. + +The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`. ## Corpus creation diff --git a/tests/iwyu_consumer/conversions.cpp b/tests/iwyu_consumer/conversions.cpp new file mode 100644 index 000000000..599da67b7 --- /dev/null +++ b/tests/iwyu_consumer/conversions.cpp @@ -0,0 +1,62 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// See values.cpp: the conversion macros and nlohmann::adl_serializer must be available through +// alone. NLOHMANN_JSON_SERIALIZE_ENUM (from the same header as +// NLOHMANN_DEFINE_TYPE_INTRUSIVE) is left out: IWYU attributes the basic_json comparison in its +// expansion to an unrelated standard header, the ambiguity described for json.hpp in cmake/ci.cmake. + +#include + +#include + +namespace ns +{ +struct person +{ + std::string name; + int age; + NLOHMANN_DEFINE_TYPE_INTRUSIVE(person, name, age) +}; + +struct legacy +{ + int value; +}; +} // namespace ns + +namespace nlohmann +{ +template<> +struct adl_serializer +{ + static void to_json(json& j, const ns::legacy& l) + { + j = l.value; + } + + static void from_json(const json& j, ns::legacy& l) + { + l.value = j.get(); + } +}; +} // namespace nlohmann + +int main() +{ + ns::person ada; + ada.name = "Ada"; + ada.age = 36; + ns::legacy one; + one.value = 1; + + const nlohmann::json j = ada; + const auto p = j.get(); + const auto l = nlohmann::json(one).get(); + return p.age - 36 + l.value - 1; +} diff --git a/tests/iwyu_consumer/literals.cpp b/tests/iwyu_consumer/literals.cpp new file mode 100644 index 000000000..7aaf58d79 --- /dev/null +++ b/tests/iwyu_consumer/literals.cpp @@ -0,0 +1,19 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// See values.cpp: the user-defined literals must be available through alone. + +#include + +using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) + +int main() +{ + const auto j = R"({"a": [1, 2]})"_json; + return j.at("/a/1"_json_pointer).get() - 2; +} diff --git a/tests/iwyu_consumer/values.cpp b/tests/iwyu_consumer/values.cpp new file mode 100644 index 000000000..1f6d00b76 --- /dev/null +++ b/tests/iwyu_consumer/values.cpp @@ -0,0 +1,30 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// Uses the library the way a client does, including only . ci_single_binaries +// runs include-what-you-use and clang-tidy's misc-include-cleaner on this file and fails if either +// asks for another nlohmann header, which would mean json.hpp does not export it (issue #5812). + +#include +#include +#include + +#include + +int main() +{ + // brace initialization (nlohmann::json, detail::json_ref) + nlohmann::json j = {{"pi", 3.141}, {"list", {1, 0, 2}}}; + const nlohmann::ordered_json o = {{"b", 1}, {"a", 2}}; + + j["parsed"] = nlohmann::json::parse(R"({"happy": true})"); + j["binary"] = nlohmann::json::binary(std::vector {0xCA, 0xFE}, 42); + + const std::string s = j.dump() + o.dump(); + return s.empty() ? 1 : 0; +} diff --git a/tests/src/float_hard_cases.hpp b/tests/src/float_hard_cases.hpp new file mode 100644 index 000000000..e579bccce --- /dev/null +++ b/tests/src/float_hard_cases.hpp @@ -0,0 +1,599 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // array +#include // uint32_t, uint64_t + +// Number tokens that are hard to round correctly, with the IEEE-754 binary64 +// and binary32 bits of their correctly rounded values (ties to even; infinity +// for an overflow, a signed zero for an underflow). +// +// For doubles and floats around 0, the smallest normal number, 1, 2^24, 2^53, +// 0.1, and the largest finite number, and for random ones, the exact midpoint +// m to the next number gives: m, m with one unit more and less in the last +// digit, m with "01" and "0...01" appended, m with trailing zeros, and m cut +// after 17 to 30 digits (rounded down and up, so that the rounding is decided +// after the 19th digit), in fixed and exponent notation, 30% of them negative. +// Tokens longer than 80 characters are left out, except for four of 700 digits +// and more. Zeros, underflow, overflow, huge exponents, and integers beyond 64 +// bits complete the set. Of the 508 tokens, 134 (as double) and 150 (as +// float) need the exact comparison with the midpoint (detail::digit_comparison()). +// +// The expected bits were computed with exact rational arithmetic in Python +// (fractions.Fraction) and cross-checked with Python's float(); strtod_l and +// strtof_l of Apple's libc and of glibc agree. Generated by +// compact_hard_cases.py 5 (with hard_cases.py), see the pull request that +// added this file. + +namespace float_hard_cases +{ + +struct hard_case +{ + const char* token; + std::uint64_t bits64; + std::uint32_t bits32; +}; + +inline const std::array& cases() +{ + static const std::array table = + { + { + {"-2.4703282292062327e-324", 0x8000000000000000u, 0x80000000u}, + {"24703282292062328e-340", 0x0000000000000001u, 0x00000000u}, + {"247032822920623272e-341", 0x0000000000000000u, 0x00000000u}, + {"-0.2470328229206232721e-323", 0x8000000000000001u, 0x80000000u}, + {"-0.24703282292062327208e-323", 0x8000000000000000u, 0x80000000u}, + {"-2.4703282292062327209e-324", 0x8000000000000001u, 0x80000000u}, + {"2.47032822920623272088e-324", 0x0000000000000000u, 0x00000000u}, + {"247032822920623272089e-344", 0x0000000000000001u, 0x00000000u}, + {"-247032822920623272088284396434e-353", 0x8000000000000000u, 0x80000000u}, + {"0.247032822920623272088284396435e-323", 0x0000000000000001u, 0x00000000u}, + {"-74109846876186981e-340", 0x8000000000000001u, 0x80000000u}, + {"0.74109846876186982e-323", 0x0000000000000002u, 0x00000000u}, + {"-0.7410984687618698162e-323", 0x8000000000000001u, 0x80000000u}, + {"-7.410984687618698163e-324", 0x8000000000000002u, 0x80000000u}, + {"7.4109846876186981626e-324", 0x0000000000000001u, 0x00000000u}, + {"-74109846876186981627e-343", 0x8000000000000002u, 0x80000000u}, + {"-741098468761869816264e-344", 0x8000000000000001u, 0x80000000u}, + {"0.741098468761869816265e-323", 0x0000000000000002u, 0x00000000u}, + {"0.741098468761869816264853189302e-323", 0x0000000000000001u, 0x00000000u}, + {"-7.41098468761869816264853189303e-324", 0x8000000000000002u, 0x80000000u}, + {"0.22250738585072006e-307", 0x000FFFFFFFFFFFFEu, 0x00000000u}, + {"2.2250738585072007e-308", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"2.225073858507200641e-308", 0x000FFFFFFFFFFFFEu, 0x00000000u}, + {"-2225073858507200642e-326", 0x800FFFFFFFFFFFFFu, 0x80000000u}, + {"22250738585072006419e-327", 0x000FFFFFFFFFFFFEu, 0x00000000u}, + {"0.2225073858507200642e-307", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"0.222507385850720064199e-307", 0x000FFFFFFFFFFFFEu, 0x00000000u}, + {"2.225073858507200642e-308", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"-2.22507385850720064199176395546e-308", 0x800FFFFFFFFFFFFEu, 0x80000000u}, + {"222507385850720064199176395547e-337", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"-2.2250738585072011e-308", 0x800FFFFFFFFFFFFFu, 0x80000000u}, + {"-22250738585072012e-324", 0x8010000000000000u, 0x80000000u}, + {"-2225073858507201136e-326", 0x800FFFFFFFFFFFFFu, 0x80000000u}, + {"0.2225073858507201137e-307", 0x0010000000000000u, 0x00000000u}, + {"0.2225073858507201136e-307", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"-2.2250738585072011361e-308", 0x8010000000000000u, 0x80000000u}, + {"2.22507385850720113605e-308", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"222507385850720113606e-328", 0x0010000000000000u, 0x00000000u}, + {"22250738585072011360574097967e-336", 0x000FFFFFFFFFFFFFu, 0x00000000u}, + {"0.222507385850720113605740979671e-307", 0x0010000000000000u, 0x00000000u}, + {"22250738585072016e-324", 0x0010000000000000u, 0x00000000u}, + {"0.22250738585072017e-307", 0x0010000000000001u, 0x00000000u}, + {"0.222507385850720163e-307", 0x0010000000000000u, 0x00000000u}, + {"2.225073858507201631e-308", 0x0010000000000001u, 0x00000000u}, + {"-2.2250738585072016301e-308", 0x8010000000000000u, 0x80000000u}, + {"22250738585072016302e-327", 0x0010000000000001u, 0x00000000u}, + {"-222507385850720163012e-328", 0x8010000000000000u, 0x80000000u}, + {"0.222507385850720163013e-307", 0x0010000000000001u, 0x00000000u}, + {"0.222507385850720163012305563795e-307", 0x0010000000000000u, 0x00000000u}, + {"-2.22507385850720163012305563796e-308", 0x8010000000000001u, 0x80000000u}, + {"0.17976931348623156E+309", 0x7FEFFFFFFFFFFFFEu, 0x7F800000u}, + {"1.7976931348623157e308", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"1.797693134862315608e308", 0x7FEFFFFFFFFFFFFEu, 0x7F800000u}, + {"-1797693134862315609e290", 0xFFEFFFFFFFFFFFFFu, 0xFF800000u}, + {"-17976931348623156083e289", 0xFFEFFFFFFFFFFFFEu, 0xFF800000u}, + {"-0.17976931348623156084E+309", 0xFFEFFFFFFFFFFFFFu, 0xFF800000u}, + {"0.179769313486231560835E+309", 0x7FEFFFFFFFFFFFFEu, 0x7F800000u}, + {"-1.79769313486231560836e308", 0xFFEFFFFFFFFFFFFFu, 0xFF800000u}, + {"1.79769313486231560835325876058e308", 0x7FEFFFFFFFFFFFFEu, 0x7F800000u}, + {"179769313486231560835325876059e279", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"1.7976931348623158e308", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"17976931348623159e292", 0x7FF0000000000000u, 0x7F800000u}, + {"1797693134862315807e290", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"0.1797693134862315808E+309", 0x7FF0000000000000u, 0x7F800000u}, + {"0.17976931348623158079E+309", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"-1.797693134862315808e308", 0xFFF0000000000000u, 0xFF800000u}, + {"1.79769313486231580793e308", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"179769313486231580794e288", 0x7FF0000000000000u, 0x7F800000u}, + {"179769313486231580793728971405e279", 0x7FEFFFFFFFFFFFFFu, 0x7F800000u}, + {"-0.179769313486231580793728971406E+309", 0xFFF0000000000000u, 0xFF800000u}, + {"100000000000000011102230246251565404236316680908203125e-53", 0x3FF0000000000000u, 0x3F800000u}, + {"-1.00000000000000011102230246251565404236316680908203126", 0xBFF0000000000001u, 0xBF800000u}, + {"1.00000000000000011102230246251565404236316680908203124e0", 0x3FF0000000000000u, 0x3F800000u}, + {"10000000000000001110223024625156540423631668090820312501e-55", 0x3FF0000000000001u, 0x3F800000u}, + {"1.00000000000000011102230246251565404236316680908203125000000000000000000001", 0x3FF0000000000001u, 0x3F800000u}, + {"10000000000000001e-16", 0x3FF0000000000000u, 0x3F800000u}, + {"1.0000000000000002", 0x3FF0000000000001u, 0x3F800000u}, + {"1.000000000000000111", 0x3FF0000000000000u, 0x3F800000u}, + {"1.000000000000000112e0", 0x3FF0000000000001u, 0x3F800000u}, + {"1.000000000000000111e0", 0x3FF0000000000000u, 0x3F800000u}, + {"-10000000000000001111e-19", 0xBFF0000000000001u, 0xBF800000u}, + {"-100000000000000011102e-20", 0xBFF0000000000000u, 0xBF800000u}, + {"-1.00000000000000011103", 0xBFF0000000000001u, 0xBF800000u}, + {"1.00000000000000011102230246251", 0x3FF0000000000000u, 0x3F800000u}, + {"1.00000000000000011102230246252e0", 0x3FF0000000000001u, 0x3F800000u}, + {"-0.999999999999999944488848768742172978818416595458984375", 0xBFF0000000000000u, 0xBF800000u}, + {"-9.99999999999999944488848768742172978818416595458984376e-1", 0xBFF0000000000000u, 0xBF800000u}, + {"999999999999999944488848768742172978818416595458984374e-54", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"0.99999999999999994448884876874217297881841659545898437501", 0x3FF0000000000000u, 0x3F800000u}, + {"9.99999999999999944488848768742172978818416595458984375000000000000000000001e-1", 0x3FF0000000000000u, 0x3F800000u}, + {"-0.99999999999999994", 0xBFEFFFFFFFFFFFFFu, 0xBF800000u}, + {"9.9999999999999995e-1", 0x3FF0000000000000u, 0x3F800000u}, + {"9.999999999999999444e-1", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"9999999999999999445e-19", 0x3FF0000000000000u, 0x3F800000u}, + {"99999999999999994448e-20", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"-0.99999999999999994449", 0xBFF0000000000000u, 0xBF800000u}, + {"0.999999999999999944488", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"-9.99999999999999944489e-1", 0xBFF0000000000000u, 0xBF800000u}, + {"9.99999999999999944488848768742e-1", 0x3FEFFFFFFFFFFFFFu, 0x3F800000u}, + {"999999999999999944488848768743e-30", 0x3FF0000000000000u, 0x3F800000u}, + {"-9.007199254740993e15", 0xC340000000000000u, 0xDA000000u}, + {"9007199254740994e0", 0x4340000000000001u, 0x5A000000u}, + {"9007199254740992", 0x4340000000000000u, 0x5A000000u}, + {"9.00719925474099301e15", 0x4340000000000001u, 0x5A000000u}, + {"9007199254740993000000000000000000001e-21", 0x4340000000000001u, 0x5A000000u}, + {"-9007199254740993.000000000000000000000000000000", 0xC340000000000000u, 0xDA000000u}, + {"90071992547409915e-1", 0x4340000000000000u, 0x5A000000u}, + {"-9007199254740991.6", 0xC340000000000000u, 0xDA000000u}, + {"9.0071992547409914e15", 0x433FFFFFFFFFFFFFu, 0x5A000000u}, + {"-9007199254740991501e-3", 0xC340000000000000u, 0xDA000000u}, + {"9007199254740991.5000000000000000000001", 0x4340000000000000u, 0x5A000000u}, + {"9.0071992547409915000000000000000000000000000000e15", 0x4340000000000000u, 0x5A000000u}, + {"0.100000000000000012490009027033011079765856266021728515625", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"1.00000000000000012490009027033011079765856266021728515626e-1", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"100000000000000012490009027033011079765856266021728515624e-57", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"0.10000000000000001249000902703301107976585626602172851562501", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"0.10000000000000001", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"1.0000000000000002e-1", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"-1.000000000000000124e-1", 0xBFB999999999999Au, 0xBDCCCCCDu}, + {"1000000000000000125e-19", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"-10000000000000001249e-20", 0xBFB999999999999Au, 0xBDCCCCCDu}, + {"0.1000000000000000125", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"0.10000000000000001249", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"1.00000000000000012491e-1", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"1.00000000000000012490009027033e-1", 0x3FB999999999999Au, 0x3DCCCCCDu}, + {"100000000000000012490009027034e-30", 0x3FB999999999999Bu, 0x3DCCCCCDu}, + {"2.45134755833537796875e14", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"-245134755833537796876e-6", 0xC2EBDE5C4164D83Au, 0xD75EF2E2u}, + {"-245134755833537.796874", 0xC2EBDE5C4164D839u, 0xD75EF2E2u}, + {"2.4513475583353779687501e14", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"245134755833537796875000000000000000000001e-27", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"245134755833537.796875000000000000000000000000000000", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"2.4513475583353779e14", 0x42EBDE5C4164D839u, 0x575EF2E2u}, + {"2451347558335378e-1", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"2451347558335377968e-4", 0x42EBDE5C4164D839u, 0x575EF2E2u}, + {"245134755833537.7969", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"245134755833537.79687", 0x42EBDE5C4164D839u, 0x575EF2E2u}, + {"2.4513475583353779688e14", 0x42EBDE5C4164D83Au, 0x575EF2E2u}, + {"181510327827821147441864013671875e-23", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"-1815103278.27821147441864013671876", 0xC1DB0C11CB91CE38u, 0xCED8608Eu}, + {"1.81510327827821147441864013671874e9", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"18151032782782114744186401367187501e-25", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"1815103278.27821147441864013671875000000000000000000001", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"-1.81510327827821147441864013671875000000000000000000000000000000e9", 0xC1DB0C11CB91CE38u, 0xCED8608Eu}, + {"18151032782782114e-7", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"1815103278.2782115", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"1815103278.278211474", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"-1.815103278278211475e9", 0xC1DB0C11CB91CE38u, 0xCED8608Eu}, + {"1.8151032782782114744e9", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"-18151032782782114745e-10", 0xC1DB0C11CB91CE38u, 0xCED8608Eu}, + {"181510327827821147441e-11", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"1815103278.27821147442", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"1815103278.27821147441864013671", 0x41DB0C11CB91CE37u, 0x4ED8608Eu}, + {"1.81510327827821147441864013672e9", 0x41DB0C11CB91CE38u, 0x4ED8608Eu}, + {"3809325632181785344", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3.809325632181785345e18", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3809325632181785343e0", 0x43CA6EB8BD69FE29u, 0x5E5375C6u}, + {"3809325632181785344.01", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3.809325632181785344000000000000000000001e18", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3809325632181785344000000000000000000000000000000e-30", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"3809325632181785300", 0x43CA6EB8BD69FE29u, 0x5E5375C6u}, + {"3.8093256321817854e18", 0x43CA6EB8BD69FE2Au, 0x5E5375C6u}, + {"4.046966549916366943359375e12", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916366943359376e-12", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916.366943359374", 0x428D7210076CE2EFu, 0x546B9080u}, + {"4.04696654991636694335937501e12", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916366943359375000000000000000000001e-33", 0x428D7210076CE2F0u, 0x546B9080u}, + {"-4046966549916.366943359375000000000000000000000000000000", 0xC28D7210076CE2F0u, 0xD46B9080u}, + {"4.0469665499163669e12", 0x428D7210076CE2EFu, 0x546B9080u}, + {"4046966549916367e-3", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916366943e-6", 0x428D7210076CE2EFu, 0x546B9080u}, + {"4046966549916.366944", 0x428D7210076CE2F0u, 0x546B9080u}, + {"4046966549916.3669433", 0x428D7210076CE2EFu, 0x546B9080u}, + {"4.0469665499163669434e12", 0x428D7210076CE2F0u, 0x546B9080u}, + {"-4.04696654991636694335e12", 0xC28D7210076CE2EFu, 0xD46B9080u}, + {"404696654991636694336e-8", 0x428D7210076CE2F0u, 0x546B9080u}, + {"28093802557000874e154", 0x63529C3B77330BDBu, 0x7F800000u}, + {"0.28093802557000875E+171", 0x63529C3B77330BDCu, 0x7F800000u}, + {"0.2809380255700087447E+171", 0x63529C3B77330BDBu, 0x7F800000u}, + {"2.809380255700087448e170", 0x63529C3B77330BDCu, 0x7F800000u}, + {"2.8093802557000874472e170", 0x63529C3B77330BDBu, 0x7F800000u}, + {"28093802557000874473e151", 0x63529C3B77330BDCu, 0x7F800000u}, + {"280938025570008744728e150", 0x63529C3B77330BDBu, 0x7F800000u}, + {"-0.280938025570008744729E+171", 0xE3529C3B77330BDCu, 0xFF800000u}, + {"-0.280938025570008744728403667979E+171", 0xE3529C3B77330BDBu, 0xFF800000u}, + {"2.8093802557000874472840366798e170", 0x63529C3B77330BDCu, 0x7F800000u}, + {"0.39523280297734525e-154", 0x1FE0F51BF17FD374u, 0x00000000u}, + {"-3.9523280297734526e-155", 0x9FE0F51BF17FD375u, 0x80000000u}, + {"3.952328029773452547e-155", 0x1FE0F51BF17FD374u, 0x00000000u}, + {"3952328029773452548e-173", 0x1FE0F51BF17FD375u, 0x00000000u}, + {"-39523280297734525478e-174", 0x9FE0F51BF17FD374u, 0x80000000u}, + {"0.39523280297734525479e-154", 0x1FE0F51BF17FD375u, 0x00000000u}, + {"-0.395232802977345254787e-154", 0x9FE0F51BF17FD374u, 0x80000000u}, + {"-3.95232802977345254788e-155", 0x9FE0F51BF17FD375u, 0x80000000u}, + {"-3.95232802977345254787245825501e-155", 0x9FE0F51BF17FD374u, 0x80000000u}, + {"395232802977345254787245825502e-184", 0x1FE0F51BF17FD375u, 0x00000000u}, + {"-1.0790205420931879e-276", 0x86A3209CA6233255u, 0x80000000u}, + {"-1079020542093188e-291", 0x86A3209CA6233256u, 0x80000000u}, + {"-1079020542093187947e-294", 0x86A3209CA6233255u, 0x80000000u}, + {"0.1079020542093187948e-275", 0x06A3209CA6233256u, 0x00000000u}, + {"0.1079020542093187947e-275", 0x06A3209CA6233255u, 0x00000000u}, + {"1.0790205420931879471e-276", 0x06A3209CA6233256u, 0x00000000u}, + {"1.07902054209318794701e-276", 0x06A3209CA6233255u, 0x00000000u}, + {"-107902054209318794702e-296", 0x86A3209CA6233256u, 0x80000000u}, + {"107902054209318794701153285302e-305", 0x06A3209CA6233255u, 0x00000000u}, + {"-0.107902054209318794701153285303e-275", 0x86A3209CA6233256u, 0x80000000u}, + {"58530471071351308e-228", 0x1413B446E6A16A3Bu, 0x00000000u}, + {"-0.58530471071351309e-211", 0x9413B446E6A16A3Cu, 0x80000000u}, + {"0.5853047107135130893e-211", 0x1413B446E6A16A3Bu, 0x00000000u}, + {"-5.853047107135130894e-212", 0x9413B446E6A16A3Cu, 0x80000000u}, + {"5.853047107135130893e-212", 0x1413B446E6A16A3Bu, 0x00000000u}, + {"58530471071351308931e-231", 0x1413B446E6A16A3Cu, 0x00000000u}, + {"-585304710713513089304e-232", 0x9413B446E6A16A3Bu, 0x80000000u}, + {"0.585304710713513089305e-211", 0x1413B446E6A16A3Cu, 0x00000000u}, + {"-0.585304710713513089304248824438e-211", 0x9413B446E6A16A3Bu, 0x80000000u}, + {"5.85304710713513089304248824439e-212", 0x1413B446E6A16A3Cu, 0x00000000u}, + {"0.19334214893983531e-78", 0x2F96ECBF1CFB10F6u, 0x00000000u}, + {"1.9334214893983532e-79", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"1.933421489398353102e-79", 0x2F96ECBF1CFB10F6u, 0x00000000u}, + {"1933421489398353103e-97", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"19334214893983531023e-98", 0x2F96ECBF1CFB10F6u, 0x00000000u}, + {"0.19334214893983531024e-78", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"0.193342148939835310231e-78", 0x2F96ECBF1CFB10F6u, 0x00000000u}, + {"1.93342148939835310232e-79", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"-1.93342148939835310231359014704e-79", 0xAF96ECBF1CFB10F6u, 0x80000000u}, + {"193342148939835310231359014705e-108", 0x2F96ECBF1CFB10F7u, 0x00000000u}, + {"2.9873358928024455e227", 0x6F2938807814E8A2u, 0x7F800000u}, + {"29873358928024456e211", 0x6F2938807814E8A3u, 0x7F800000u}, + {"298733589280244551e210", 0x6F2938807814E8A2u, 0x7F800000u}, + {"0.2987335892802445511E+228", 0x6F2938807814E8A3u, 0x7F800000u}, + {"0.29873358928024455109E+228", 0x6F2938807814E8A2u, 0x7F800000u}, + {"2.987335892802445511e227", 0x6F2938807814E8A3u, 0x7F800000u}, + {"-2.98733589280244551098e227", 0xEF2938807814E8A2u, 0xFF800000u}, + {"-298733589280244551099e207", 0xEF2938807814E8A3u, 0xFF800000u}, + {"298733589280244551098081559931e198", 0x6F2938807814E8A2u, 0x7F800000u}, + {"0.298733589280244551098081559932E+228", 0x6F2938807814E8A3u, 0x7F800000u}, + {"7.0064923216240853e-46", 0x3690000000000000u, 0x00000000u}, + {"70064923216240854e-62", 0x3690000000000000u, 0x00000001u}, + {"7006492321624085354e-64", 0x3690000000000000u, 0x00000000u}, + {"-0.7006492321624085355e-45", 0xB690000000000000u, 0x80000001u}, + {"0.70064923216240853546e-45", 0x3690000000000000u, 0x00000000u}, + {"7.0064923216240853547e-46", 0x3690000000000000u, 0x00000001u}, + {"7.00649232162408535461e-46", 0x3690000000000000u, 0x00000000u}, + {"-700649232162408535462e-66", 0xB690000000000000u, 0x80000001u}, + {"-700649232162408535461864791644e-75", 0xB690000000000000u, 0x80000000u}, + {"0.700649232162408535461864791645e-45", 0x3690000000000000u, 0x00000001u}, + {"21019476964872256e-61", 0x36A8000000000000u, 0x00000001u}, + {"0.21019476964872257e-44", 0x36A8000000000000u, 0x00000002u}, + {"-0.2101947696487225606e-44", 0xB6A8000000000000u, 0x80000001u}, + {"-2.101947696487225607e-45", 0xB6A8000000000000u, 0x80000002u}, + {"2.1019476964872256063e-45", 0x36A8000000000000u, 0x00000001u}, + {"21019476964872256064e-64", 0x36A8000000000000u, 0x00000002u}, + {"-210194769648722560638e-65", 0xB6A8000000000000u, 0x80000001u}, + {"-0.210194769648722560639e-44", 0xB6A8000000000000u, 0x80000002u}, + {"-0.210194769648722560638559437493e-44", 0xB6A8000000000000u, 0x80000001u}, + {"2.10194769648722560638559437494e-45", 0x36A8000000000000u, 0x00000002u}, + {"0.11754941406275178e-37", 0x380FFFFFA0000000u, 0x007FFFFEu}, + {"1.1754941406275179e-38", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"-1.175494140627517859e-38", 0xB80FFFFFA0000000u, 0x807FFFFEu}, + {"117549414062751786e-55", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"-11754941406275178592e-57", 0xB80FFFFFA0000000u, 0x807FFFFEu}, + {"0.11754941406275178593e-37", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"0.117549414062751785924e-37", 0x380FFFFFA0000000u, 0x007FFFFEu}, + {"1.17549414062751785925e-38", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"-1.17549414062751785924617589866e-38", 0xB80FFFFFA0000000u, 0x807FFFFEu}, + {"117549414062751785924617589867e-67", 0x380FFFFFA0000000u, 0x007FFFFFu}, + {"1.1754942807573642e-38", 0x380FFFFFDFFFFFFFu, 0x007FFFFFu}, + {"11754942807573643e-54", 0x380FFFFFE0000000u, 0x00800000u}, + {"1175494280757364291e-56", 0x380FFFFFE0000000u, 0x007FFFFFu}, + {"0.1175494280757364292e-37", 0x380FFFFFE0000000u, 0x00800000u}, + {"0.11754942807573642917e-37", 0x380FFFFFE0000000u, 0x007FFFFFu}, + {"-1.1754942807573642918e-38", 0xB80FFFFFE0000000u, 0x80800000u}, + {"-1.17549428075736429172e-38", 0xB80FFFFFE0000000u, 0x807FFFFFu}, + {"-117549428075736429173e-58", 0xB80FFFFFE0000000u, 0x80800000u}, + {"117549428075736429172788299103e-67", 0x380FFFFFE0000000u, 0x007FFFFFu}, + {"0.117549428075736429172788299104e-37", 0x380FFFFFE0000000u, 0x00800000u}, + {"11754944208872107e-54", 0x3810000010000000u, 0x00800000u}, + {"-0.11754944208872108e-37", 0xB810000010000000u, 0x80800001u}, + {"0.1175494420887210724e-37", 0x3810000010000000u, 0x00800000u}, + {"1.175494420887210725e-38", 0x3810000010000000u, 0x00800001u}, + {"1.1754944208872107242e-38", 0x3810000010000000u, 0x00800000u}, + {"-11754944208872107243e-57", 0xB810000010000000u, 0x80800001u}, + {"-11754944208872107242e-57", 0xB810000010000000u, 0x80800000u}, + {"0.117549442088721072421e-37", 0x3810000010000000u, 0x00800001u}, + {"0.11754944208872107242095900834e-37", 0x3810000010000000u, 0x00800000u}, + {"-1.17549442088721072420959008341e-38", 0xB810000010000000u, 0x80800001u}, + {"340282336497324057985868971510891282432", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"-3.40282336497324057985868971510891282433e38", 0xC7EFFFFFD0000000u, 0xFF7FFFFFu}, + {"340282336497324057985868971510891282431e0", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324057985868971510891282432.01", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"3.40282336497324057985868971510891282432000000000000000000001e38", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"340282336497324057985868971510891282432000000000000000000000000000000e-30", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324050000000000000000000000", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"3.4028233649732406e38", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"3.402823364973240579e38", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324058e21", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"34028233649732405798e19", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324057990000000000000000000", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"340282336497324057985000000000000000000", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"3.40282336497324057986e38", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"3.4028233649732405798586897151e38", 0x47EFFFFFD0000000u, 0x7F7FFFFEu}, + {"340282336497324057985868971511e9", 0x47EFFFFFD0000000u, 0x7F7FFFFFu}, + {"3.40282356779733661637539395458142568448e38", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"-340282356779733661637539395458142568449e0", 0xC7EFFFFFF0000000u, 0xFF800000u}, + {"340282356779733661637539395458142568447", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"3.4028235677973366163753939545814256844801e38", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"340282356779733661637539395458142568448000000000000000000001e-21", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"340282356779733661637539395458142568448.000000000000000000000000000000", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"3.4028235677973366e38", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"-34028235677973367e22", 0xC7EFFFFFF0000000u, 0xFF800000u}, + {"3402823567797336616e20", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"340282356779733661700000000000000000000", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"340282356779733661630000000000000000000", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"3.4028235677973366164e38", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"3.40282356779733661637e38", 0x47EFFFFFF0000000u, 0x7F7FFFFFu}, + {"340282356779733661638e18", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"-340282356779733661637539395458e9", 0xC7EFFFFFF0000000u, 0xFF7FFFFFu}, + {"340282356779733661637539395459000000000", 0x47EFFFFFF0000000u, 0x7F800000u}, + {"-1000000059604644775390625e-24", 0xBFF0000010000000u, 0xBF800000u}, + {"-1.000000059604644775390626", 0xBFF0000010000000u, 0xBF800001u}, + {"-1.000000059604644775390624e0", 0xBFF0000010000000u, 0xBF800000u}, + {"100000005960464477539062501e-26", 0x3FF0000010000000u, 0x3F800001u}, + {"1.000000059604644775390625000000000000000000001", 0x3FF0000010000000u, 0x3F800001u}, + {"1.000000059604644775390625000000000000000000000000000000e0", 0x3FF0000010000000u, 0x3F800000u}, + {"-10000000596046447e-16", 0xBFF0000010000000u, 0xBF800000u}, + {"1.0000000596046448", 0x3FF0000010000000u, 0x3F800001u}, + {"1.000000059604644775", 0x3FF0000010000000u, 0x3F800000u}, + {"-1.000000059604644776e0", 0xBFF0000010000000u, 0xBF800001u}, + {"-1.0000000596046447753e0", 0xBFF0000010000000u, 0xBF800000u}, + {"10000000596046447754e-19", 0x3FF0000010000000u, 0x3F800001u}, + {"100000005960464477539e-20", 0x3FF0000010000000u, 0x3F800000u}, + {"-1.0000000596046447754", 0xBFF0000010000000u, 0xBF800001u}, + {"0.9999999701976776123046875", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"9.999999701976776123046876e-1", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"9999999701976776123046874e-25", 0x3FEFFFFFF0000000u, 0x3F7FFFFFu}, + {"0.999999970197677612304687501", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"-9.999999701976776123046875000000000000000000001e-1", 0xBFEFFFFFF0000000u, 0xBF800000u}, + {"-9999999701976776123046875000000000000000000000000000000e-55", 0xBFEFFFFFF0000000u, 0xBF800000u}, + {"0.99999997019767761", 0x3FEFFFFFF0000000u, 0x3F7FFFFFu}, + {"-9.9999997019767762e-1", 0xBFEFFFFFF0000000u, 0xBF800000u}, + {"-9.999999701976776123e-1", 0xBFEFFFFFF0000000u, 0xBF7FFFFFu}, + {"9999999701976776124e-19", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"9999999701976776123e-19", 0x3FEFFFFFF0000000u, 0x3F7FFFFFu}, + {"0.99999997019767761231", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"-0.999999970197677612304", 0xBFEFFFFFF0000000u, 0xBF7FFFFFu}, + {"9.99999970197677612305e-1", 0x3FEFFFFFF0000000u, 0x3F800000u}, + {"-1.6777217e7", 0xC170000010000000u, 0xCB800000u}, + {"16777218e0", 0x4170000020000000u, 0x4B800001u}, + {"16777216", 0x4170000000000000u, 0x4B800000u}, + {"-1.677721701e7", 0xC17000001028F5C3u, 0xCB800001u}, + {"-16777217000000000000000000001e-21", 0xC170000010000000u, 0xCB800001u}, + {"16777217.000000000000000000000000000000", 0x4170000010000000u, 0x4B800000u}, + {"167772155e-1", 0x416FFFFFF0000000u, 0x4B800000u}, + {"16777215.6", 0x416FFFFFF3333333u, 0x4B800000u}, + {"1.67772154e7", 0x416FFFFFECCCCCCDu, 0x4B7FFFFFu}, + {"16777215501e-3", 0x416FFFFFF0083127u, 0x4B800000u}, + {"-16777215.5000000000000000000001", 0xC16FFFFFF0000000u, 0xCB800000u}, + {"-1.67772155000000000000000000000000000000e7", 0xC16FFFFFF0000000u, 0xCB800000u}, + {"0.1000000052154064178466796875", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"1.000000052154064178466796876e-1", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"-1000000052154064178466796874e-28", 0xBFB99999B0000000u, 0xBDCCCCCDu}, + {"0.100000005215406417846679687501", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"1.000000052154064178466796875000000000000000000001e-1", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"-1000000052154064178466796875000000000000000000000000000000e-58", 0xBFB99999B0000000u, 0xBDCCCCCEu}, + {"-0.10000000521540641", 0xBFB99999AFFFFFFFu, 0xBDCCCCCDu}, + {"-1.0000000521540642e-1", 0xBFB99999B0000000u, 0xBDCCCCCEu}, + {"1.000000052154064178e-1", 0x3FB99999B0000000u, 0x3DCCCCCDu}, + {"-1000000052154064179e-19", 0xBFB99999B0000000u, 0xBDCCCCCEu}, + {"10000000521540641784e-20", 0x3FB99999B0000000u, 0x3DCCCCCDu}, + {"0.10000000521540641785", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"0.100000005215406417846", 0x3FB99999B0000000u, 0x3DCCCCCDu}, + {"1.00000005215406417847e-1", 0x3FB99999B0000000u, 0x3DCCCCCEu}, + {"5.429001220703125e3", 0x40B5350050000000u, 0x45A9A802u}, + {"-5429001220703126e-12", 0xC0B5350050000001u, 0xC5A9A803u}, + {"-5429.001220703124", 0xC0B535004FFFFFFFu, 0xC5A9A802u}, + {"5.42900122070312501e3", 0x40B5350050000000u, 0x45A9A803u}, + {"5429001220703125000000000000000000001e-33", 0x40B5350050000000u, 0x45A9A803u}, + {"-5429.001220703125000000000000000000000000000000", 0xC0B5350050000000u, 0xC5A9A802u}, + {"503719056e0", 0x41BE062490000000u, 0x4DF03124u}, + {"503719057", 0x41BE062491000000u, 0x4DF03125u}, + {"5.03719055e8", 0x41BE06248F000000u, 0x4DF03124u}, + {"50371905601e-2", 0x41BE062490028F5Cu, 0x4DF03125u}, + {"503719056.000000000000000000001", 0x41BE062490000000u, 0x4DF03125u}, + {"5.03719056000000000000000000000000000000e8", 0x41BE062490000000u, 0x4DF03124u}, + {"-92331620", 0xC196037990000000u, 0xCCB01BCCu}, + {"9.233163e7", 0x41960379B8000000u, 0x4CB01BCEu}, + {"9233161e1", 0x4196037968000000u, 0x4CB01BCBu}, + {"92331620.1", 0x4196037990666666u, 0x4CB01BCDu}, + {"9.233162000000000000000000001e7", 0x4196037990000000u, 0x4CB01BCDu}, + {"9233162000000000000000000000000000000e-29", 0x4196037990000000u, 0x4CB01BCCu}, + {"3.002458625e6", 0x4146E82D50000000u, 0x4A37416Au}, + {"3002458626e-3", 0x4146E82D5020C49Cu, 0x4A37416Bu}, + {"3002458.624", 0x4146E82D4FDF3B64u, 0x4A37416Au}, + {"-3.00245862501e6", 0xC146E82D500053E3u, 0xCA37416Bu}, + {"3002458625000000000000000000001e-24", 0x4146E82D50000000u, 0x4A37416Bu}, + {"3002458.625000000000000000000000000000000", 0x4146E82D50000000u, 0x4A37416Au}, + {"-1095485584696182596504479582065262592e1", 0xC7A07BA830000000u, 0xFD03DD42u}, + {"10954855846961825965044795820652625930", 0x47A07BA830000000u, 0x7D03DD42u}, + {"1.095485584696182596504479582065262591e37", 0x47A07BA830000000u, 0x7D03DD41u}, + {"109548558469618259650447958206526259201e-1", 0x47A07BA830000000u, 0x7D03DD42u}, + {"-10954855846961825965044795820652625920.00000000000000000001", 0xC7A07BA830000000u, 0xFD03DD42u}, + {"1.095485584696182596504479582065262592000000000000000000000000000000e37", 0x47A07BA830000000u, 0x7D03DD42u}, + {"10954855846961825e21", 0x47A07BA830000000u, 0x7D03DD41u}, + {"10954855846961826000000000000000000000", 0x47A07BA830000000u, 0x7D03DD42u}, + {"10954855846961825960000000000000000000", 0x47A07BA830000000u, 0x7D03DD41u}, + {"1.095485584696182597e37", 0x47A07BA830000000u, 0x7D03DD42u}, + {"1.0954855846961825965e37", 0x47A07BA830000000u, 0x7D03DD41u}, + {"-10954855846961825966e18", 0xC7A07BA830000000u, 0xFD03DD42u}, + {"10954855846961825965e18", 0x47A07BA830000000u, 0x7D03DD41u}, + {"10954855846961825965100000000000000000", 0x47A07BA830000000u, 0x7D03DD42u}, + {"10954855846961825965044795820600000000", 0x47A07BA830000000u, 0x7D03DD41u}, + {"-1.09548558469618259650447958207e37", 0xC7A07BA830000000u, 0xFD03DD42u}, + {"1.6449216019182103706535606608388384863861375606575165875256061553955078126e-21", 0x3B9F125A50000000u, 0x1CF892D3u}, + {"-16449216019182103706535606608388384863861375606575165875256061553955078124e-94", 0xBB9F125A50000000u, 0x9CF892D2u}, + {"-0.0000000000000000000016449216019182103", 0xBB9F125A50000000u, 0x9CF892D2u}, + {"-1.6449216019182104e-21", 0xBB9F125A50000000u, 0x9CF892D3u}, + {"-1.64492160191821037e-21", 0xBB9F125A50000000u, 0x9CF892D2u}, + {"-1644921601918210371e-39", 0xBB9F125A50000000u, 0x9CF892D3u}, + {"-16449216019182103706e-40", 0xBB9F125A50000000u, 0x9CF892D2u}, + {"0.0000000000000000000016449216019182103707", 0x3B9F125A50000000u, 0x1CF892D3u}, + {"0.00000000000000000000164492160191821037065", 0x3B9F125A50000000u, 0x1CF892D2u}, + {"1.64492160191821037066e-21", 0x3B9F125A50000000u, 0x1CF892D3u}, + {"1.64492160191821037065356066083e-21", 0x3B9F125A50000000u, 0x1CF892D2u}, + {"164492160191821037065356066084e-50", 0x3B9F125A50000000u, 0x1CF892D3u}, + {"6.565061509609222412109375e-1", 0x3FE5021930000000u, 0x3F2810CAu}, + {"6565061509609222412109376e-25", 0x3FE5021930000000u, 0x3F2810CAu}, + {"0.6565061509609222412109374", 0x3FE5021930000000u, 0x3F2810C9u}, + {"6.56506150960922241210937501e-1", 0x3FE5021930000000u, 0x3F2810CAu}, + {"6565061509609222412109375000000000000000000001e-46", 0x3FE5021930000000u, 0x3F2810CAu}, + {"-0.6565061509609222412109375000000000000000000000000000000", 0xBFE5021930000000u, 0xBF2810CAu}, + {"6.5650615096092224e-1", 0x3FE5021930000000u, 0x3F2810C9u}, + {"-65650615096092225e-17", 0xBFE5021930000000u, 0xBF2810CAu}, + {"6565061509609222412e-19", 0x3FE5021930000000u, 0x3F2810C9u}, + {"0.6565061509609222413", 0x3FE5021930000000u, 0x3F2810CAu}, + {"0.65650615096092224121", 0x3FE5021930000000u, 0x3F2810C9u}, + {"6.5650615096092224122e-1", 0x3FE5021930000000u, 0x3F2810CAu}, + {"-6.5650615096092224121e-1", 0xBFE5021930000000u, 0xBF2810C9u}, + {"656506150960922241211e-21", 0x3FE5021930000000u, 0x3F2810CAu}, + {"18014627239033005156980393746124491372029297053813934326171875e-77", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"0.00000000000000018014627239033005156980393746124491372029297053813934326171876", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"-1.8014627239033005156980393746124491372029297053813934326171874e-16", 0xBCA9F63970000000u, 0xA54FB1CBu}, + {"1801462723903300515698039374612449137202929705381393432617187501e-79", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"18014627239033005e-32", 0x3CA9F63970000000u, 0x254FB1CBu}, + {"0.00000000000000018014627239033006", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"0.0000000000000001801462723903300515", 0x3CA9F63970000000u, 0x254FB1CBu}, + {"-1.801462723903300516e-16", 0xBCA9F63970000000u, 0xA54FB1CCu}, + {"-1.8014627239033005156e-16", 0xBCA9F63970000000u, 0xA54FB1CBu}, + {"18014627239033005157e-35", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"180146272390330051569e-36", 0x3CA9F63970000000u, 0x254FB1CBu}, + {"-0.00000000000000018014627239033005157", 0xBCA9F63970000000u, 0xA54FB1CCu}, + {"0.000000000000000180146272390330051569803937461", 0x3CA9F63970000000u, 0x254FB1CBu}, + {"1.80146272390330051569803937462e-16", 0x3CA9F63970000000u, 0x254FB1CCu}, + {"0.05534819327294826507568359375", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"-5.534819327294826507568359376e-2", 0xBFAC569930000000u, 0xBD62B4CAu}, + {"5534819327294826507568359374e-29", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"-0.0553481932729482650756835937501", 0xBFAC569930000000u, 0xBD62B4CAu}, + {"5.534819327294826507568359375000000000000000000001e-2", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"5534819327294826507568359375000000000000000000000000000000e-59", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"0.055348193272948265", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"5.5348193272948266e-2", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"5.534819327294826507e-2", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"5534819327294826508e-20", 0x3FAC569930000000u, 0x3D62B4CAu}, + {"55348193272948265075e-21", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"-0.055348193272948265076", 0xBFAC569930000000u, 0xBD62B4CAu}, + {"0.0553481932729482650756", 0x3FAC569930000000u, 0x3D62B4C9u}, + {"-5.53481932729482650757e-2", 0xBFAC569930000000u, 0xBD62B4CAu}, + {"5.179692133247783258005389047985340416e36", 0x478F2C9450000000u, 0x7C7964A2u}, + {"5179692133247783258005389047985340417e0", 0x478F2C9450000000u, 0x7C7964A3u}, + {"5179692133247783258005389047985340415", 0x478F2C9450000000u, 0x7C7964A2u}, + {"-5.17969213324778325800538904798534041601e36", 0xC78F2C9450000000u, 0xFC7964A3u}, + {"-5179692133247783258005389047985340416000000000000000000001e-21", 0xC78F2C9450000000u, 0xFC7964A3u}, + {"-5179692133247783258005389047985340416.000000000000000000000000000000", 0xC78F2C9450000000u, 0xFC7964A2u}, + {"5.1796921332477832e36", 0x478F2C9450000000u, 0x7C7964A2u}, + {"51796921332477833e20", 0x478F2C9450000000u, 0x7C7964A3u}, + {"-5179692133247783258e18", 0xC78F2C9450000000u, 0xFC7964A2u}, + {"5179692133247783259000000000000000000", 0x478F2C9450000000u, 0x7C7964A3u}, + {"5179692133247783258000000000000000000", 0x478F2C9450000000u, 0x7C7964A2u}, + {"5.1796921332477832581e36", 0x478F2C9450000000u, 0x7C7964A3u}, + {"-5.179692133247783258e36", 0xC78F2C9450000000u, 0xFC7964A2u}, + {"-517969213324778325801e16", 0xC78F2C9450000000u, 0xFC7964A3u}, + {"517969213324778325800538904798e7", 0x478F2C9450000000u, 0x7C7964A2u}, + {"5179692133247783258005389047990000000", 0x478F2C9450000000u, 0x7C7964A3u}, + { + "0.22250738585072011360574097967091319759348195463516456480234261097248222220210769455165295239081350" + "8791414915891303962110687008643869459464552765720740782062174337998814106326732925355228688137214901" + "2981122451451889849057222307285255133155755015914397476397983411801999323962548289017107081850690630" + "6666559949382757725720157630626906633326475653000092458883164330377797918696120494973903778297049050" + "5108060994073026293712895895000358379996720725430436028407889577179615094551674824347103070260914462" + "1572289880258182545180325707018860872113128079512233426288368622321503775666622503982534335974568884" + "4239002654981983854879482922068947216898310996983658468140228542433306603398508864458040010349339704" + "2756718644338377048603786162277173854562306587467901408672332763671875e-307", 0x0010000000000000u, 0x00000000u + }, + { + "2.22507385850720113605740979670913197593481954635164564802342610972482222202107694551652952390813508" + "7914149158913039621106870086438694594645527657207407820621743379988141063267329253552286881372149012" + "9811224514518898490572223072852551331557550159143974763979834118019993239625482890171070818506906306" + "6665599493827577257201576306269066333264756530000924588831643303777979186961204949739037782970490505" + "1080609940730262937128958950003583799967207254304360284078895771796150945516748243471030702609144621" + "5722898802581825451803257070188608721131280795122334262883686223215037756666225039825343359745688844" + "2390026549819838548794829220689472168983109969836584681402285424333066033985088644580400103493397042" + "756718644338377048603786162277173854562306587467901408672332763671875000000000000000000001e-308", 0x0010000000000000u, 0x00000000u + }, + { + "0.11754942807573642917278829910357665133228589927589904276829631184250030649651730385585324256680905" + "8189392089843750000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000e-37", 0x380FFFFFE0000000u, 0x00800000u + }, + { + "1175494280757364291727882991035766513322858992758990427682963118425003064965173038558532425668090581" + "8939208984375000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000001e-751", 0x380FFFFFE0000000u, 0x00800000u + }, + {"0", 0x0000000000000000u, 0x00000000u}, + {"-0", 0x8000000000000000u, 0x80000000u}, + {"0.0", 0x0000000000000000u, 0x00000000u}, + {"-0.0", 0x8000000000000000u, 0x80000000u}, + {"0e999999999999999999999", 0x0000000000000000u, 0x00000000u}, + {"-0.000e-99999", 0x8000000000000000u, 0x80000000u}, + {"1e-400", 0x0000000000000000u, 0x00000000u}, + {"-1e-400", 0x8000000000000000u, 0x80000000u}, + {"1e400", 0x7FF0000000000000u, 0x7F800000u}, + {"-1e400", 0xFFF0000000000000u, 0xFF800000u}, + {"1e-50", 0x358DEE7A4AD4B81Fu, 0x00000000u}, + {"-1e-50", 0xB58DEE7A4AD4B81Fu, 0x80000000u}, + {"1e39", 0x48078287F49C4A1Du, 0x7F800000u}, + {"-1e39", 0xC8078287F49C4A1Du, 0xFF800000u}, + {"1e99999999999999999999999999", 0x7FF0000000000000u, 0x7F800000u}, + {"1e-99999999999999999999999999", 0x0000000000000000u, 0x00000000u}, + {"1e0000000000000000000000000000000000000000308", 0x7FE1CCF385EBC8A0u, 0x7F800000u}, + {"123456789012345678901234567890e-30", 0x3FBF9ADD3746F65Fu, 0x3DFCD6EAu}, + {"18446744073709551615", 0x43F0000000000000u, 0x5F800000u}, + {"18446744073709551616", 0x43F0000000000000u, 0x5F800000u}, + {"-9223372036854775808", 0xC3E0000000000000u, 0xDF000000u}, + {"-9223372036854775809", 0xC3E0000000000000u, 0xDF000000u}, + } + }; + return table; +} + +} // namespace float_hard_cases diff --git a/tests/src/fuzzer-parse_bjdata.cpp b/tests/src/fuzzer-parse_bjdata.cpp index 73e41d545..fe9bb10b1 100644 --- a/tests/src/fuzzer-parse_bjdata.cpp +++ b/tests/src/fuzzer-parse_bjdata.cpp @@ -51,6 +51,11 @@ Furthermore, it reads data with a SAX parser that recovers from every error and checks that the events are balanced, that reading ends, and that it reports an error exactly when from_bjdata() fails (see #3989). +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ @@ -58,6 +63,19 @@ drivers. #include "fuzzer_common.hpp" #include "fuzzer-recovering_checker.hpp" +namespace +{ +// json::from_bjdata for check_input_adapters() +struct read_bjdata +{ + template + json operator()(Args&& ... args) const + { + return json::from_bjdata(std::forward(args)...); + } +}; +} // namespace + // value-stable comparison for the round-trip checks below; see the note // above on why this compares dump()s rather than the json values directly static bool is_value_stable(const json& lhs, const json& rhs) @@ -139,6 +157,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || !recovered_without_errors); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bjdata()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_bon8.cpp b/tests/src/fuzzer-parse_bon8.cpp index d2332dfec..12450c3e6 100644 --- a/tests/src/fuzzer-parse_bon8.cpp +++ b/tests/src/fuzzer-parse_bon8.cpp @@ -17,38 +17,33 @@ array data, it performs the following steps: - j2 = from_bon8(vec) - assert(to_bon8(j2) == vec) -It also checks that reading the data from a stream, which reads strings byte by -byte, gives the same value or error as reading it from contiguous memory, which -copies strings in bulk. - Furthermore, it reads data with a SAX parser that recovers from every error and checks that the events are balanced, that reading ends, and that it reports an error exactly when from_bon8() fails (see #3989). +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ -#include #include "fuzzer_common.hpp" #include "fuzzer-recovering_checker.hpp" namespace { -// the serialization of the value read from @a input, or the error message -template -std::string read_bon8(InputType&& input) +// json::from_bon8 for check_input_adapters() +struct read_bon8 { - try + template + json operator()(Args&& ... args) const { - const auto vec = json::to_bon8(json::from_bon8(std::forward(input))); - return {vec.begin(), vec.end()}; + return json::from_bon8(std::forward(args)...); } - catch (const json::exception& e) - { - return e.what(); - } -} +}; } // namespace // see http://llvm.org/docs/LibFuzzer.html @@ -57,12 +52,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) // recover from all errors, reading from memory and from a stream const bool recovered_without_errors = check_recovering_parse(data, size, json::input_format_t::bon8).errors == 0; - // contiguous and stream input must be read alike - { - std::istringstream stream(std::string(reinterpret_cast(data), size)); - assert(read_bon8(std::vector(data, data + size)) == read_bon8(stream)); - } - std::vector const vec1(data, data + size); // step 0: parse input without exceptions @@ -119,6 +108,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || !recovered_without_errors); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bon8()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_bson.cpp b/tests/src/fuzzer-parse_bson.cpp index fccab0a04..6b5667f6b 100644 --- a/tests/src/fuzzer-parse_bson.cpp +++ b/tests/src/fuzzer-parse_bson.cpp @@ -21,6 +21,11 @@ Furthermore, it reads data with a SAX parser that recovers from every error and checks that the events are balanced, that reading ends, and that it reports an error exactly when from_bson() fails (see #3989). +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ @@ -28,6 +33,19 @@ drivers. #include "fuzzer_common.hpp" #include "fuzzer-recovering_checker.hpp" +namespace +{ +// json::from_bson for check_input_adapters() +struct read_bson +{ + template + json operator()(Args&& ... args) const + { + return json::from_bson(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -90,6 +108,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || !recovered_without_errors); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bson()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_cbor.cpp b/tests/src/fuzzer-parse_cbor.cpp index 9ecea7924..cf97f4972 100644 --- a/tests/src/fuzzer-parse_cbor.cpp +++ b/tests/src/fuzzer-parse_cbor.cpp @@ -21,6 +21,11 @@ Furthermore, it reads data with a SAX parser that recovers from every error and checks that the events are balanced, that reading ends, and that it reports an error exactly when from_cbor() fails (see #3989). +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ @@ -28,6 +33,19 @@ drivers. #include "fuzzer_common.hpp" #include "fuzzer-recovering_checker.hpp" +namespace +{ +// json::from_cbor for check_input_adapters() +struct read_cbor +{ + template + json operator()(Args&& ... args) const + { + return json::from_cbor(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -90,6 +108,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || !recovered_without_errors); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_cbor()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_json.cpp b/tests/src/fuzzer-parse_json.cpp index 246f2b544..8f7fc0fd1 100644 --- a/tests/src/fuzzer-parse_json.cpp +++ b/tests/src/fuzzer-parse_json.cpp @@ -17,6 +17,12 @@ array data, it performs the following steps: - j2 = parse(s1) - s2 = serialize(j2) - assert(s1 == s2) +- for ignore_comments = false and true: assert(parse(data) and accept(data) + give the same result for a stream, a non-contiguous iterator range, and a + FILE* as for the pointer range) + +The pointer range takes the bulk fast paths, the other input adapters read byte +by byte; see check_input_adapters(). Furthermore, it parses data with a SAX parser that recovers from every error and checks that the events are balanced, that parsing ends, and that valid @@ -29,6 +35,32 @@ drivers. #include "fuzzer_common.hpp" #include "fuzzer-recovering_checker.hpp" +namespace +{ +// json::parse and json::accept for check_input_adapters() +struct parse_json +{ + bool ignore_comments; + + template + json operator()(Args&& ... args) const + { + return json::parse(std::forward(args)..., nullptr, true, ignore_comments); + } +}; + +struct accept_json +{ + bool ignore_comments; + + template + bool operator()(Args&& ... args) const + { + return json::accept(std::forward(args)..., ignore_comments); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -88,6 +120,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + for (const bool ignore_comments : + { + false, true + }) + { + check_input_adapters(data, size, parse_json{ignore_comments}); + check_input_adapters(data, size, accept_json{ignore_comments}); + } + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_msgpack.cpp b/tests/src/fuzzer-parse_msgpack.cpp index 1abc86eba..0bce235f7 100644 --- a/tests/src/fuzzer-parse_msgpack.cpp +++ b/tests/src/fuzzer-parse_msgpack.cpp @@ -21,6 +21,11 @@ Furthermore, it reads data with a SAX parser that recovers from every error and checks that the events are balanced, that reading ends, and that it reports an error exactly when from_msgpack() fails (see #3989). +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ @@ -28,6 +33,19 @@ drivers. #include "fuzzer_common.hpp" #include "fuzzer-recovering_checker.hpp" +namespace +{ +// json::from_msgpack for check_input_adapters() +struct read_msgpack +{ + template + json operator()(Args&& ... args) const + { + return json::from_msgpack(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -90,6 +108,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || !recovered_without_errors); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_msgpack()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_ubjson.cpp b/tests/src/fuzzer-parse_ubjson.cpp index fec0bbcfe..d14370138 100644 --- a/tests/src/fuzzer-parse_ubjson.cpp +++ b/tests/src/fuzzer-parse_ubjson.cpp @@ -30,6 +30,11 @@ Furthermore, it reads data with a SAX parser that recovers from every error and checks that the events are balanced, that reading ends, and that it reports an error exactly when from_ubjson() fails (see #3989). +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ @@ -37,6 +42,19 @@ drivers. #include "fuzzer_common.hpp" #include "fuzzer-recovering_checker.hpp" +namespace +{ +// json::from_ubjson for check_input_adapters() +struct read_ubjson +{ + template + json operator()(Args&& ... args) const + { + return json::from_ubjson(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -109,6 +127,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || !recovered_without_errors); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_ubjson()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer_common.hpp b/tests/src/fuzzer_common.hpp index 4b6c9289c..8761b3a35 100644 --- a/tests/src/fuzzer_common.hpp +++ b/tests/src/fuzzer_common.hpp @@ -11,6 +11,14 @@ #pragma once #include // assert +#include // size_t +#include // uint8_t +#include // FILE, fmemopen, fclose +#include // list +#include // istringstream +#include // string +#include // forward +#include // vector #include using nlohmann::json; // NOLINT(google-global-names-in-headers): shared by all fuzzer drivers @@ -49,3 +57,89 @@ json parse_without_exceptions(Parse parse, bool& threw) } return {}; } + +// the outcome of reading an input with exceptions: the serialization of the +// value, or the id, message, and (for parse errors) byte of the exception +struct read_outcome +{ + std::string value{}; + int id = 0; + std::string what{}; + std::size_t byte = 0; + + bool operator==(const read_outcome& rhs) const + { + return value == rhs.value && id == rhs.id && what == rhs.what && byte == rhs.byte; + } +}; + +inline std::string outcome_value(const json& j) +{ + return j.dump(-1, ' ', false, json::error_handler_t::keep); +} + +inline std::string outcome_value(bool b) +{ + return b ? "true" : "false"; +} + +template +read_outcome read_with(const Read& parse, Args&& ... args) +{ + read_outcome result; + try + { + result.value = outcome_value(parse(std::forward(args)...)); + } + catch (const json::parse_error& e) + { + result.id = e.id; + result.what = e.what(); + result.byte = e.byte; + } + catch (const json::exception& e) + { + result.id = e.id; + result.what = e.what(); + } + return result; +} + +// The contiguous input (a pointer range) takes the bulk fast paths; the other +// input adapters read byte by byte. Reads @a data through each of them and +// checks that all give the same value, or the same exception with the same +// id, message, and byte position. +// +// @a parse is called with either an iterator range or a single input (stream +// or FILE*) and returns the parsed value (or the result of accept()). +template +void check_input_adapters(const std::uint8_t* data, std::size_t size, const Read& parse) +{ + const read_outcome expected = read_with(parse, data, data + size); + + // stream adapter + { + std::istringstream stream(std::string(reinterpret_cast(data), size)); + assert(read_with(parse, stream) == expected); + } + + // iterator adapter on a non-contiguous range + { + const std::list list(data, data + size); + assert(read_with(parse, list.begin(), list.end()) == expected); + } + +#if defined(__unix__) || defined(__APPLE__) + // FILE* adapter; fmemopen may refuse an empty buffer + { + std::vector buffer(data, data + size); + std::FILE* file = buffer.empty() ? nullptr : fmemopen(buffer.data(), buffer.size(), "rb"); + if (file != nullptr) + { + const read_outcome actual = read_with(parse, file); + std::fclose(file); + assert(actual == expected); + } + } +#endif +} diff --git a/tests/src/unit-algorithms.cpp b/tests/src/unit-algorithms.cpp index 90d6a08c7..2a9506590 100644 --- a/tests/src/unit-algorithms.cpp +++ b/tests/src/unit-algorithms.cpp @@ -9,6 +9,7 @@ #include "doctest_compatibility.h" #include +#include #include using nlohmann::json; diff --git a/tests/src/unit-binary_formats.cpp b/tests/src/unit-binary_formats.cpp index 03dbc94da..333252190 100644 --- a/tests/src/unit-binary_formats.cpp +++ b/tests/src/unit-binary_formats.cpp @@ -38,7 +38,7 @@ TEST_CASE("Binary Formats") const auto ubjson_2_size = json::to_ubjson(j, true).size(); const auto ubjson_3_size = json::to_ubjson(j, true, true).size(); - CHECK(json_size == 2090303); + CHECK(json_size == 2090234); CHECK(bjdata_1_size == 1112030); CHECK(bjdata_2_size == 1224148); CHECK(bjdata_3_size == 1224148); @@ -51,16 +51,16 @@ TEST_CASE("Binary Formats") CHECK(ubjson_3_size == 1169069); CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0)); - CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(53.199)); - CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(58.563)); - CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(58.563)); - CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(50.509)); - CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(85.849)); - CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(50.497)); - CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(50.526)); - CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(53.199)); - CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(58.563)); - CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(55.928)); + CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(53.201)); + CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(58.565)); + CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(58.565)); + CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(50.511)); + CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(85.853)); + CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(50.499)); + CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(50.528)); + CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(53.201)); + CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(58.565)); + CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(55.930)); } SECTION("twitter.json") diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 860af81d6..05cd2bf1d 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -13,16 +13,20 @@ using nlohmann::json; #include // array -#include // FLT_EVAL_METHOD #include // uint32_t, uint64_t +#include // snprintf #include // strtod #include // memcpy #include // numeric_limits +#include // map +#include // mt19937 #include // stringstream #include // string #include // pair #include // vector +#include "float_hard_cases.hpp" + namespace { // shortcut to scan a string literal @@ -258,7 +262,7 @@ TEST_CASE("lexer number fast path") "123456789012345678901234567890", // huge -> float "0.30000000000000004", "2.2250738585072014e-308", "1e308", // high-precision / wide-exponent values that exercise the - // std::from_chars (Eisel-Lemire) path beyond the Clinger subset + // Eisel-Lemire path beyond the Clinger subset "1.7976931348623157e308", "1.2345678901234567e-250", "9007199254740993", "5e-324", "1e-320" }; @@ -280,20 +284,18 @@ TEST_CASE("lexer number fast path") } } - SECTION("significant-digit gate for the Clinger fast path") + SECTION("significant digits around Clinger's fast path") { - // Clinger's fast path needs a significand below 2^53, so it cannot - // succeed once the mantissa has 17 or more significant digits (the - // significand would be at least 10^16). The lexer skips the attempt - // there. That is only allowed to save work: every value must still come - // out bit-exactly, and both scanners must agree. In particular the gate - // must not fire for tokens whose leading zeros merely look like extra - // digits - "0.1234567890123456" has 16 significant digits, not 17. + // Clinger's fast path needs a significand of at most 2^53, which + // tokens with 17 or more significant digits exceed. The conversion + // splits the token at the positions the scanners recorded, so leading + // zeros must not count as digits - "0.1234567890123456" has 16 + // significant digits, not 17 - and both scanners must agree. const std::vector numbers = { "1234567890123456", // 16 significant digits - "12345678901234567", // 17 -> attempt skipped - "123456789012345678", // 18 -> attempt skipped + "12345678901234567", // 17 + "123456789012345678", // 18 "0.1234567890123456", // 16: the leading "0" is not significant "0.12345678901234567", // 17 "0.00000000000000001", // 1, in a long token @@ -664,46 +666,323 @@ TEST_CASE("lexer string fast path") } } -TEST_CASE("parse_float_fast declines what it cannot convert exactly") +TEST_CASE("lexer escape fast path") { - // The lexer only hands well-formed numbers to parse_float_fast, so the - // malformed ones below can only be passed to it directly. Declining is - // always safe: the caller then falls back to a slower, exact conversion. - const auto fast = [](const std::string & s, double & out) + // json::accept() never throws, so this section stays covered without + // exceptions; it pins which of the cases below are valid/invalid and + // checks the contiguous and streaming paths agree on that classification. + SECTION("accept() parity") { - return nlohmann::detail::parse_float_fast(s.data(), s.data() + s.size(), out); + const std::vector> cases = + { + {"\\u0041", true}, {"\\u00e4", true}, {"\\u00E4", true}, + {"\\uD83D\\uDE00", true}, + {"\\u12", false}, {"\\u12G4", false}, {"\\uXYZW", false}, + {"\\uD800", false}, {"\\uD800A", false}, {"\\uD800\\u0041", false}, + {"\\uDC00", false}, {"\\u", false} + }; + + for (const auto& c : cases) + { + for (const std::size_t offset : + { + std::size_t{0}, std::size_t{9} + }) + { + const std::string doc = "[\"" + std::string(offset, 'a') + c.first + "\"]"; + CAPTURE(doc) + CHECK(json::accept(doc) == c.second); + std::stringstream ss(doc); + CHECK(json::accept(ss) == c.second); + } + } + } + +#if !defined(JSON_NOEXCEPTION) + // the full outcome of parsing @a doc: the parsed value, or the exact + // error message, so a mismatch in either is caught + const auto outcome = [](const std::string & doc, bool streaming) -> std::string + { + try + { + if (streaming) + { + std::stringstream ss(doc); + const json j = json::parse(ss); + return j.dump(); + } + const json j = json::parse(doc); + return j.dump(); + } + catch (const json::exception& e) + { + return {e.what()}; + } }; - double out = 0; -#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0 - // without true double precision, the fast path declines everything - CHECK_FALSE(fast("1.5", out)); -#else - CHECK(fast("1.5", out)); - CHECK(out == 1.5); - CHECK(fast("+2.5e1", out)); - CHECK(out == 25.0); - CHECK(fast("-25E-1", out)); - CHECK(out == -2.5); - CHECK(fast("1e", out)); - CHECK(out == 1.0); + SECTION("contiguous vs streaming parity") + { + const std::vector escapes = + { + "\\u0041", // "A" + "\\u00e4", // "ä" (lowercase hex) + "\\u00E4", // "ä" (uppercase hex) + "\\uD83D\\uDE00", // valid surrogate pair (an emoji) + "\\u12", // truncated: only 2 hex digits before the closing quote + "\\u12G4", // invalid hex digit at the 3rd position + "\\uXYZW", // all 4 bytes invalid + "\\uD800", // lone high surrogate, string ends right after + "\\uD800A", // high surrogate not followed by another \u escape + "\\uD800\\u0041", // high surrogate followed by \u, but not a low surrogate + "\\uDC00", // lone low surrogate + "\\u", // '\u' with nothing after (closing quote right away) + }; + + // once at the start of the string and once past the first 8-byte SWAR + // word of the outer string_bulk_run, so the escape is reached both + // right after the opening quote and mid-run + for (const auto& escape : escapes) + { + for (const std::size_t offset : + { + std::size_t{0}, std::size_t{9} + }) + { + const std::string doc = "[\"" + std::string(offset, 'a') + escape + "\"]"; + CAPTURE(doc) + CHECK(outcome(doc, false) == outcome(doc, true)); + } + + // the escape is the last thing before end of input: no closing + // quote at all + const std::string truncated_doc = "[\"" + escape; + CAPTURE(truncated_doc) + CHECK(outcome(truncated_doc, false) == outcome(truncated_doc, true)); + } + } + + SECTION("truncated \\u escape at every distance from the end of input") + { + // ia.bulk_remaining() must correctly report fewer than 4 bytes for + // every possible count of trailing hex-looking bytes (0, 1, 2, or 3) + // before end of input, so the fast path declines and the byte path + // alone reports the "must be followed by 4 hex digits" error, at the + // same position, in every case + for (const std::string& tail : + { + std::string{}, std::string("1"), std::string("12"), std::string("123") + }) + { + const std::string doc = "[\"\\u" + tail; + CAPTURE(doc) + CHECK(outcome(doc, false) == outcome(doc, true)); + CHECK(outcome(doc, false).find("must be followed by 4 hex digits") != std::string::npos); + } + } + + SECTION("invalid hex digit at every position of the 4") + { + // the fast path must decline for *any* invalid byte among the 4, not + // just the first, and the byte path must then stop at exactly that + // position - same as it always has + for (std::size_t bad_pos = 0; bad_pos < 4; ++bad_pos) + { + std::string digits = "1234"; + digits[bad_pos] = 'g'; // not a hex digit + const std::string doc = "[\"\\u" + digits + "\"]"; + CAPTURE(doc) + CHECK(outcome(doc, false) == outcome(doc, true)); + CHECK(outcome(doc, false).find("must be followed by 4 hex digits") != std::string::npos); + } + } + + SECTION("random escapes") + { + // A seeded PRNG builds the 4 bytes following `\u` from a mix of hex + // digits and non-hex bytes, at varying distances from the start of + // the string, to compare the two scanners on many more shapes than + // are practical to enumerate by hand. + std::mt19937 gen(7654321); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + const std::string hex_alphabet = "0123456789AaBbCcDdEeFf"; + std::uniform_int_distribution pick_hex(0, hex_alphabet.size() - 1); + std::uniform_int_distribution pick_byte(1, 255); // never NUL + std::uniform_int_distribution pick_is_hex(0, 4); // 4-in-5 chance of a hex digit + std::uniform_int_distribution pick_offset(0, 12); + + std::vector mismatches; + for (int iter = 0; iter < 3000; ++iter) + { + std::string digits; + for (int i = 0; i < 4; ++i) + { + if (pick_is_hex(gen) != 0) + { + digits += hex_alphabet[pick_hex(gen)]; + } + else + { + char c = static_cast(pick_byte(gen)); + if (c == '"' || c == '\\') + { + // keep the string well-formed apart from the escape + // itself, so any mismatch is attributable to the \u + // handling and not to an unrelated quote/escape + c = 'z'; + } + digits += c; + } + } + const std::string doc = "[\"" + std::string(pick_offset(gen), 'a') + "\\u" + digits + "\"]"; + if (outcome(doc, false) != outcome(doc, true)) + { + mismatches.push_back(doc); + } + } + CAPTURE(mismatches) + CHECK(mismatches.empty()); + } #endif +} - // not a number - CHECK_FALSE(fast("", out)); - CHECK_FALSE(fast("-", out)); - CHECK_FALSE(fast(".", out)); - CHECK_FALSE(fast("1.2.3", out)); - CHECK_FALSE(fast("1x", out)); - CHECK_FALSE(fast("1e+", out)); - CHECK_FALSE(fast("1e1x", out)); +namespace +{ +// the index of the decimal point (or npos) and of the end of the mantissa of a +// number token, which the lexer records while scanning it +std::pair float_token_layout(const std::string& s) +{ + std::size_t dot = std::string::npos; + std::size_t mantissa_end = s.size(); + for (std::size_t i = 0; i < s.size(); ++i) + { + if (s[i] == '.') + { + dot = i; + } + else if (s[i] == 'e' || s[i] == 'E') + { + mantissa_end = i; + break; + } + } + return {dot, mantissa_end}; +} - // numbers that are not represented exactly on the fast path - CHECK_FALSE(fast("12345678901234567890", out)); - CHECK_FALSE(fast("1e10000", out)); - CHECK_FALSE(fast("9007199254740993", out)); - CHECK_FALSE(fast("1e23", out)); - CHECK_FALSE(fast("1e-23", out)); +template +FloatType parse_native(const std::string& s) +{ + const auto layout = float_token_layout(s); + return nlohmann::detail::parse_float_native(s.data(), s.data() + s.size(), layout.first, layout.second); +} + +std::uint64_t bits_of(double d) +{ + std::uint64_t b = 0; + std::memcpy(&b, &d, sizeof(b)); + return b; +} + +std::uint32_t bits_of(float f) +{ + std::uint32_t b = 0; + std::memcpy(&b, &f, sizeof(b)); + return b; +} + +std::uint64_t native_bits64(const std::string& s) +{ + return bits_of(parse_native(s)); +} + +std::uint32_t native_bits32(const std::string& s) +{ + return bits_of(parse_native(s)); +} +} // namespace + +TEST_CASE("parse_float_native rounds correctly") +{ + SECTION("double") + { + CHECK(native_bits64("1.5") == 0x3FF8000000000000u); + CHECK(native_bits64("0.1") == 0x3FB999999999999Au); + CHECK(native_bits64("-0.0") == 0x8000000000000000u); + CHECK(native_bits64("0e999999999999999999999") == 0u); + // 2^53 + 1 is exactly between two doubles: ties to even, unless more digits follow + CHECK(native_bits64("9007199254740993") == 0x4340000000000000u); + CHECK(native_bits64("9007199254740993.0000000000000000001") == 0x4340000000000001u); + CHECK(native_bits64("9007199254740992.9999999999999999999") == 0x4340000000000000u); + // 1 + 2^-53 exactly (a tie), and one unit in the 55th digit around it + CHECK(native_bits64("1.00000000000000011102230246251565404236316680908203125") == 0x3FF0000000000000u); + CHECK(native_bits64("1.00000000000000011102230246251565404236316680908203126") == 0x3FF0000000000001u); + CHECK(native_bits64("1.00000000000000011102230246251565404236316680908203124") == 0x3FF0000000000000u); + // subnormal and overflow boundaries + CHECK(native_bits64("2.4703282292062327e-324") == 0u); + CHECK(native_bits64("2.4703282292062328e-324") == 1u); + CHECK(native_bits64("2.2250738585072011e-308") == 0x000FFFFFFFFFFFFFu); + CHECK(native_bits64("2.2250738585072012e-308") == 0x0010000000000000u); + CHECK(native_bits64("1.7976931348623157e308") == 0x7FEFFFFFFFFFFFFFu); + CHECK(native_bits64("1.7976931348623159e308") == 0x7FF0000000000000u); + CHECK(native_bits64("-1e400") == 0xFFF0000000000000u); + CHECK(native_bits64("-1e-400") == 0x8000000000000000u); + // exponents and zeros far beyond the range cancel out + CHECK(native_bits64("0." + std::string(1000, '0') + "1e1001") == 0x3FF0000000000000u); + CHECK(native_bits64("1" + std::string(1000, '0') + "e-1000") == 0x3FF0000000000000u); + CHECK(native_bits64("1e-99999999999999999999999") == 0u); + CHECK(native_bits64("1E+99999999999999999999999") == 0x7FF0000000000000u); + // more digits than any midpoint has (769): only whether a nonzero digit follows matters + const std::string tie = "1.00000000000000011102230246251565404236316680908203125"; + CHECK(native_bits64(tie + std::string(800, '0')) == 0x3FF0000000000000u); + CHECK(native_bits64(tie + std::string(800, '0') + "1") == 0x3FF0000000000001u); + } + + SECTION("float") + { + CHECK(native_bits32("1.5") == 0x3FC00000u); + CHECK(native_bits32("0.1") == 0x3DCCCCCDu); + CHECK(native_bits32("-0.0") == 0x80000000u); + // 2^24 + 1 is exactly between two floats + CHECK(native_bits32("16777217") == 0x4B800000u); + CHECK(native_bits32("16777217.000000000000000000001") == 0x4B800001u); + CHECK(native_bits32("16777218.999999999999999999999") == 0x4B800001u); + CHECK(native_bits32("16777219") == 0x4B800002u); + // subnormal and overflow boundaries + CHECK(native_bits32("3.4028235677973366e38") == 0x7F7FFFFFu); + CHECK(native_bits32("3.4028235677973367e38") == 0x7F800000u); + CHECK(native_bits32("7.006492321624085e-46") == 0u); + CHECK(native_bits32("7.006492321624086e-46") == 1u); + CHECK(native_bits32("1.1754942e-38") == 0x007FFFFFu); + CHECK(native_bits32("-1.17549435e-38") == 0x80800000u); + CHECK(native_bits32("1e39") == 0x7F800000u); + CHECK(native_bits32("-1e-50") == 0x80000000u); + // not rounded through double: its double would round to another float + CHECK(native_bits32("1.00000005960464477539062500000000001") == 0x3F800001u); + CHECK(native_bits32("9007199254740993") == 0x5A000000u); + } + + SECTION("the conversion shared with other parsers") + { + // convert_float() gives the lexer's results, for every type + const std::vector tokens = + { + "0", "-0.0", "1.5", "0.1", "1e-400", "-2.5E+3", "123456789012345678901234567890", + "9007199254740993.0000000000000000001", "4.9406564584124654e-324" + }; + using float_json = nlohmann::basic_json; + using long_double_json = nlohmann::basic_json; + for (const auto& t : tokens) + { + CAPTURE(t) + const auto layout = float_token_layout(t); + const char* const first = t.data(); + const char* const last = first + t.size(); + const auto d = nlohmann::detail::convert_float(first, last, layout.first, layout.second); + const auto f = nlohmann::detail::convert_float(first, last, layout.first, layout.second); + const auto ld = nlohmann::detail::convert_float(first, last, layout.first, layout.second); + CHECK(bits_of(d) == bits_of(json::parse(t).get())); + CHECK(bits_of(f) == bits_of(float_json::parse(t).get())); + CHECK(ld == long_double_json::parse(t).get()); + } + } } namespace @@ -807,40 +1086,6 @@ std::size_t big_bit_length(const big_uint& a) } return n; } - -std::uint64_t bits_of(double d) -{ - std::uint64_t b = 0; - std::memcpy(&b, &d, sizeof(b)); - return b; -} - -bool eisel_lemire(const std::string& s, double& out) -{ - return nlohmann::detail::parse_float_eisel_lemire(s.data(), s.data() + s.size(), out); -} - -// significant digits of a token, without trailing zeros -std::size_t significant_digits(const std::string& s) -{ - std::string digits; - for (const char c : s) - { - if (c == 'e' || c == 'E') - { - break; - } - if (c >= '0' && c <= '9' && !(digits.empty() && c == '0')) - { - digits += c; - } - } - while (!digits.empty() && digits.back() == '0') - { - digits.pop_back(); - } - return digits.size(); -} } // namespace TEST_CASE("Eisel-Lemire float conversion") @@ -938,6 +1183,10 @@ TEST_CASE("Eisel-Lemire float conversion") const int k = i % 64; const std::uint64_t x = (std::uint64_t{1} << k) | (a & ((std::uint64_t{1} << k) - 1)); CHECK(nlohmann::detail::count_leading_zeros(x) == 63 - k); + + // the mirror image: bit k set, arbitrary bits above it + const std::uint64_t y = (std::uint64_t{1} << k) | (k == 63 ? 0 : (a << (k + 1))); + CHECK(nlohmann::detail::count_trailing_zeros(y) == k); } } @@ -1268,26 +1517,33 @@ TEST_CASE("Eisel-Lemire float conversion") for (const auto& c : known) { CAPTURE(c.first) - double out = 0; - if (eisel_lemire(c.first, out)) - { - CHECK(bits_of(out) == c.second); - } - else - { - // only tokens with more than 19 significant digits are left to - // strtod: those whose value lies too close to a tie - CHECK(significant_digits(c.first) > 19); - } + CHECK(native_bits64(c.first) == c.second); } } + SECTION("binary32") + { + using binary32 = nlohmann::detail::ieee_binary_format<24>; + CHECK(nlohmann::detail::eisel_lemire(0, 1) == 0x3F800000u); + CHECK(nlohmann::detail::eisel_lemire(-1, 1) == 0x3DCCCCCDu); + CHECK(nlohmann::detail::eisel_lemire(-1, 15) == 0x3FC00000u); + CHECK(nlohmann::detail::eisel_lemire(0, 16777217) == 0x4B800000u); // tie, to even + CHECK(nlohmann::detail::eisel_lemire(0, 16777219) == 0x4B800002u); // tie, to even + CHECK(nlohmann::detail::eisel_lemire(-45, 1) == 0x00000001u); + CHECK(nlohmann::detail::eisel_lemire(-46, 7) == 0x00000000u); + CHECK(nlohmann::detail::eisel_lemire(-46, 8) == 0x00000001u); + CHECK(nlohmann::detail::eisel_lemire(-65, 9999999999999999999u) == 0x00000000u); + CHECK(nlohmann::detail::eisel_lemire(20, 3402823466385288598u) == 0x7F7FFFFFu); + CHECK(nlohmann::detail::eisel_lemire(20, 3402823669209384635u) == 0x7F800000u); + CHECK(nlohmann::detail::eisel_lemire(39, 1) == 0x7F800000u); + CHECK(nlohmann::detail::eisel_lemire(-5, 0) == 0x00000000u); + } + SECTION("round trip") { - // every double written by to_chars and read back, also with trailing - // digits that make the token longer than 19 digits + // every double written by to_chars and read back, and its 17-digit + // form with trailing digits that make the token longer than 19 digits std::uint64_t state = 5295; - std::size_t declined = 0; for (int i = 0; i < 200000; ++i) { state ^= state << 13u; @@ -1309,30 +1565,51 @@ TEST_CASE("Eisel-Lemire float conversion") const char* end = nlohmann::detail::to_chars(buffer.data(), buffer.data() + buffer.size(), d); const std::string token(buffer.data(), static_cast(end - buffer.data())); CAPTURE(token) - double out = 0; - REQUIRE(eisel_lemire(token, out)); - CHECK(bits_of(out) == b); + CHECK(native_bits64(token) == b); - // insert digits before the exponent: the value moves by far less - // than the distance to the rounding boundary, so it must not change - std::string longer = token; + // insert digits before the exponent of the 17-digit form: that + // form lies strictly inside the rounding interval of the double + // (the shortest one may lie on its boundary), and the digits move + // it by far less than the distance to the boundary, so the value + // must not change + std::array digits17{}; + static_cast(std::snprintf(digits17.data(), digits17.size(), "%.17g", d)); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + std::string longer = digits17.data(); const std::size_t e = longer.find('e'); const std::size_t dot = longer.find('.'); const std::string extra = dot == std::string::npos ? ".000000000000000000001" : "000000000000000000001"; longer.insert(e == std::string::npos ? longer.size() : e, extra); CAPTURE(longer) - if (eisel_lemire(longer, out)) + CHECK(native_bits64(longer) == b); + } + } + + SECTION("round trip, binary32") + { + std::uint32_t state = 5295; + for (int i = 0; i < 100000; ++i) + { + state ^= state << 13u; + state ^= state >> 17u; + state ^= state << 5u; + std::uint32_t b = state; + if ((b & 0x7F800000u) == 0x7F800000u) { - CHECK(bits_of(out) == b); + continue; // infinity or NaN } - else + if (i % 4 == 0) { - // w and w + 1 round differently: only when the value is very - // close to a rounding boundary - ++declined; + b &= 0x807FFFFFu; // subnormals } + float f = 0; + std::memcpy(&f, &b, sizeof(f)); + + std::array buffer{}; + const char* end = nlohmann::detail::to_chars(buffer.data(), buffer.data() + buffer.size(), f); + const std::string token(buffer.data(), static_cast(end - buffer.data())); + CAPTURE(token) + CHECK(native_bits32(token) == b); } - CHECK(declined < 1000); // 107 of the 200,000 } SECTION("used by the lexer") @@ -1346,3 +1623,184 @@ TEST_CASE("Eisel-Lemire float conversion") "[json.exception.out_of_range.406] number overflow parsing '1.7976931348623159e308'", json::out_of_range&); } } + +namespace +{ +using float_json = nlohmann::basic_json; + +// the bits of the float that parse() gives for a token, via both scanners; +// the value must be the same for both +template +void check_parse(const std::string& token, Bits expected, Bits infinity) +{ + std::stringstream stream(token); + if ((expected & ~(Bits{1} << ((8 * sizeof(Bits)) - 1))) == infinity) + { + Json _; + CHECK_THROWS_WITH_AS(_ = Json::parse(token), ("[json.exception.out_of_range.406] number overflow parsing '" + token + "'").c_str(), typename Json::out_of_range&); + CHECK_THROWS_WITH_AS(_ = Json::parse(stream), ("[json.exception.out_of_range.406] number overflow parsing '" + token + "'").c_str(), typename Json::out_of_range&); + return; + } + const Json contiguous = Json::parse(token); + const Json streamed = Json::parse(stream); + if (contiguous.is_number_float()) // not an integer that fits + { + CHECK(bits_of(contiguous.template get()) == expected); + CHECK(bits_of(streamed.template get()) == expected); + } + else + { + CHECK(streamed.is_number_integer()); + } +} +} // namespace + +TEST_CASE("float conversion of hard cases") +{ + // see float_hard_cases.hpp + for (const auto& c : float_hard_cases::cases()) + { + const std::string token = c.token; + CAPTURE(token) + CHECK(native_bits64(token) == c.bits64); + CHECK(native_bits32(token) == c.bits32); + check_parse(token, c.bits64, std::uint64_t{0x7FF0000000000000u}); + check_parse(token, c.bits32, std::uint32_t{0x7F800000u}); + } +} + +TEST_CASE("float overflow and underflow in the parser") +{ + SECTION("double") + { + check_parse("1.7976931348623157e308", std::uint64_t{0x7FEFFFFFFFFFFFFFu}, std::uint64_t{0x7FF0000000000000u}); + check_parse("1.7976931348623159e308", std::uint64_t{0x7FF0000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("-1e309", std::uint64_t{0xFFF0000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("1" + std::string(400, '0'), std::uint64_t{0x7FF0000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("1e99999999999999999999", std::uint64_t{0x7FF0000000000000u}, std::uint64_t{0x7FF0000000000000u}); + // an underflow gives a zero with the sign of the token + check_parse("1e-400", std::uint64_t{0}, std::uint64_t{0x7FF0000000000000u}); + check_parse("-1e-400", std::uint64_t{0x8000000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("-2.4703282292062327e-324", std::uint64_t{0x8000000000000000u}, std::uint64_t{0x7FF0000000000000u}); + check_parse("0." + std::string(400, '0') + "1", std::uint64_t{0}, std::uint64_t{0x7FF0000000000000u}); + } + + SECTION("float") + { + check_parse("3.4028234e38", std::uint32_t{0x7F7FFFFFu}, std::uint32_t{0x7F800000u}); + check_parse("3.4028236e38", std::uint32_t{0x7F800000u}, std::uint32_t{0x7F800000u}); + check_parse("-1e39", std::uint32_t{0xFF800000u}, std::uint32_t{0x7F800000u}); + check_parse("1e-46", std::uint32_t{0}, std::uint32_t{0x7F800000u}); + check_parse("-1e-46", std::uint32_t{0x80000000u}, std::uint32_t{0x7F800000u}); + check_parse("-7.006492321624085e-46", std::uint32_t{0x80000000u}, std::uint32_t{0x7F800000u}); + check_parse("-7.006492321624086e-46", std::uint32_t{0x80000001u}, std::uint32_t{0x7F800000u}); + } +} + +TEST_CASE("string scanning kernels") +{ + // the word-at-a-time kernels must stop exactly where a byte-by-byte scan + // stops, for any content, length, and alignment + const auto reference_special = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n && !nlohmann::detail::is_string_special(data[i])) + { + ++i; + } + return i; + }; + const auto reference_copyable = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n && nlohmann::detail::is_ascii_copyable(data[i])) + { + ++i; + } + return i; + }; + const auto reference_bulk_run = [](const unsigned char* data, std::size_t n) + { + std::size_t i = 0; + while (i < n) + { + if (data[i] < 0x80u) + { + if (nlohmann::detail::is_string_special(data[i])) + { + break; + } + ++i; + continue; + } + const std::size_t seq = nlohmann::detail::validate_one_utf8(data + i, n - i); + if (seq == 0) + { + break; + } + i += seq; + } + return i; + }; + + // pieces: ordinary ASCII, stops, DEL, well-formed sequences of every + // length, and ill-formed or truncated ones + const std::vector pieces = + { + "a", "Z", " ", "~", "0123456789", "\"", "\\", std::string(1, '\0'), "\n", "\x1F", "\x7F", + "\xC3\xA4", "\xE2\x82\xAC", "\xE6\x97\xA5\xE6\x9C\xAC", "\xF0\x9F\x98\x80", "\xED\x9F\xBF", + "\x80", "\xC0\x80", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF4\x90\x80\x80", "\xFF", + }; + std::uint64_t state = 5295; + const auto next = [&state]() + { + state ^= state << 13u; + state ^= state >> 7u; + state ^= state << 17u; + return state; + }; + // the upper half as a 32-bit value: converts to std::size_t implicitly on + // every platform (a cast of std::uint64_t is useless where both are the + // same type, and required where std::size_t is 32 bits wide) + const auto next_small = [&next]() + { + return static_cast(next() >> 32u); + }; + for (int round = 0; round < 100000; ++round) + { + // mostly ordinary text, so that runs span several words + std::string text(next_small() % 8u, '.'); + const std::size_t count = next_small() % 12u; + for (std::size_t k = 0; k < count; ++k) + { + const std::size_t p = (next() % 4 == 0) ? next_small() % pieces.size() : 0; + text += pieces[p]; + text += std::string(next_small() % 10u, 'x'); + } + const auto* data = reinterpret_cast(text.data()); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + for (std::size_t offset = 0; offset < 3 && offset <= text.size(); ++offset) + { + const std::size_t n = text.size() - offset; + CAPTURE(text) + CAPTURE(offset) + CHECK(nlohmann::detail::find_string_special(data + offset, n) == reference_special(data + offset, n)); + CHECK(nlohmann::detail::find_ascii_copyable_run(data + offset, n) == reference_copyable(data + offset, n)); + CHECK(nlohmann::detail::scalar_string_bulk_run(data + offset, n) == reference_bulk_run(data + offset, n)); + } + } + + // the trailing-zero count, whichever implementation the compiler gets + for (int k = 0; k < 64; ++k) + { + const std::uint64_t bit = std::uint64_t{1} << k; + CHECK(nlohmann::detail::count_trailing_zeros(bit) == k); + CHECK(nlohmann::detail::count_trailing_zeros(bit | (bit << 1u) | 0x8000000000000000u) == k); + } + + // eight bytes as a little-endian word, at any alignment + const std::array bytes = {{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF}}; + CHECK(nlohmann::detail::read_eight_bytes(bytes.data()) == 0x0807060504030201u); + CHECK(nlohmann::detail::read_eight_bytes(bytes.data() + 1) == 0x0908070605040302u); + CHECK(nlohmann::detail::read_eight_bytes(bytes.data() + 8) == 0xFF0F0E0D0C0B0A09u); + CHECK(nlohmann::detail::read_eight_bytes(reinterpret_cast(bytes.data()) + 3) == 0x0B0A090807060504u); +} diff --git a/tests/src/unit-comparison.cpp b/tests/src/unit-comparison.cpp index 1e409bfb0..4ac43320b 100644 --- a/tests/src/unit-comparison.cpp +++ b/tests/src/unit-comparison.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -1149,3 +1150,157 @@ TEST_CASE("operator<=> of binary values with a different subtype does not depend } } #endif + +TEST_CASE("comparison of scalars (inline fast path)") +{ + const json::number_integer_t neg = -1; + const json::number_integer_t pos = 5; + const json::number_unsigned_t big = (std::numeric_limits::max)(); + + SECTION("same type") + { + CHECK(json(pos) == json(pos)); + CHECK(json(pos) != json(neg)); + CHECK(json(neg) < json(pos)); + CHECK_FALSE(json(pos) < json(neg)); + CHECK_FALSE(json(pos) < json(pos)); + CHECK(json(pos) <= json(pos)); + + CHECK(json(1u) == json(1u)); + CHECK(json(1u) < json(2u)); + CHECK_FALSE(json(2u) < json(1u)); + + CHECK(json(1.5) == json(1.5)); + CHECK(json(1.5) < json(2.5)); + CHECK_FALSE(json(2.5) < json(1.5)); + CHECK(json(-0.0) == json(0.0)); + } + + SECTION("mixed number types") + { + // negative signed vs unsigned must not wrap around + CHECK(json(neg) < json(1u)); + CHECK(json(1u) > json(neg)); + CHECK(json(neg) != json(big)); + CHECK(json(neg) < json(big)); + CHECK_FALSE(json(big) < json(neg)); + + CHECK(json(pos) == json(5u)); + CHECK(json(5u) == json(pos)); + CHECK(json(pos) < json(6u)); + CHECK(json(pos) == json(5.0)); + CHECK(json(5.0) == json(pos)); + CHECK(json(5u) == json(5.0)); + CHECK(json(neg) < json(0.5)); + CHECK(json(0.5) < json(1u)); + } + + SECTION("boolean and null") + { + CHECK(json(true) == json(true)); + CHECK(json(true) != json(false)); + CHECK(json(false) < json(true)); + CHECK_FALSE(json(true) < json(false)); + CHECK_FALSE(json(true) < json(true)); + + CHECK(json(nullptr) == json(nullptr)); + CHECK_FALSE(json(nullptr) < json(nullptr)); + CHECK(json(nullptr) <= json(nullptr)); + CHECK(json(nullptr) < json(false)); + CHECK(json(nullptr) != json(false)); + CHECK(json(nullptr) != json(0)); + } + + SECTION("string vs number") + { + CHECK(json("5") != json(5)); + CHECK(json(5) != json("5")); + CHECK(json(5) < json("5")); // number < string by type order + CHECK_FALSE(json("5") < json(5)); + CHECK(json(1.5) < json("")); + CHECK(json("a") == json("a")); + CHECK(json("a") < json("b")); + } + + SECTION("NaN") + { + const json nan = std::numeric_limits::quiet_NaN(); + const json one = 1.0; + + CHECK_FALSE(nan == nan); + CHECK(nan != nan); + CHECK_FALSE(nan < nan); + CHECK_FALSE(nan <= nan); + CHECK_FALSE(nan >= nan); + CHECK_FALSE(nan > nan); + + CHECK_FALSE(nan == one); + CHECK_FALSE(one == nan); + CHECK_FALSE(nan < one); + CHECK_FALSE(one < nan); + CHECK_FALSE(nan <= one); + CHECK_FALSE(one <= nan); + CHECK_FALSE(nan >= one); + CHECK_FALSE(one >= nan); + + // NaN against other number types + CHECK_FALSE(nan == json(1)); + CHECK_FALSE(json(1) == nan); + CHECK_FALSE(nan < json(1u)); + CHECK_FALSE(json(1u) < nan); + +#if JSON_HAS_THREE_WAY_COMPARISON + CHECK((nan <=> nan) == std::partial_ordering::unordered); // *NOPAD* + CHECK((nan <=> one) == std::partial_ordering::unordered); // *NOPAD* + CHECK((one <=> nan) == std::partial_ordering::unordered); // *NOPAD* + CHECK((one <=> json(2.0)) == std::partial_ordering::less); // *NOPAD* + CHECK((json(2.0) <=> one) == std::partial_ordering::greater); // *NOPAD* + CHECK((one <=> one) == std::partial_ordering::equivalent); // *NOPAD* + CHECK((json(neg) <=> json(1u)) == std::partial_ordering::less); // *NOPAD* + CHECK((json(nullptr) <=> json(nullptr)) == std::partial_ordering::equivalent); // *NOPAD* + CHECK((json(true) <=> json(false)) == std::partial_ordering::greater); // *NOPAD* +#endif + } + + SECTION("discarded") + { + const json d(json::value_t::discarded); + + CHECK_FALSE(d == d); + CHECK_FALSE(d < d); + CHECK_FALSE(d == json(1)); + CHECK_FALSE(json(1) == d); + CHECK_FALSE(d < json(1)); + CHECK_FALSE(json(1) < d); + CHECK_FALSE(d == json(nullptr)); + CHECK_FALSE(json(nullptr) == d); + +#if JSON_HAS_THREE_WAY_COMPARISON + CHECK((d <=> d) == std::partial_ordering::unordered); // *NOPAD* + CHECK((d <=> json(1)) == std::partial_ordering::unordered); // *NOPAD* + CHECK((json(1) <=> d) == std::partial_ordering::unordered); // *NOPAD* +#endif + } + + SECTION("scalar overloads") + { + CHECK(json(5) == 5); + CHECK(5 == json(5)); + CHECK(json(5) < 6); + CHECK(4 < json(5)); + CHECK(json(1.5) == 1.5); + CHECK(json(true) == true); + CHECK(json(5) != 6); + CHECK(json(1u) == 1.0); + } + + SECTION("structured values still compare") + { + CHECK(json({1, 2}) == json({1, 2})); + CHECK(json({1, 2}) < json({1, 3})); + CHECK(json({{"a", 1}}) == json({{"a", 1}})); + CHECK(json::binary({1, 2}) == json::binary({1, 2})); + CHECK(json(1) < json::array()); + CHECK(json::object() < json::array()); + } +} diff --git a/tests/src/unit-constructor2.cpp b/tests/src/unit-constructor2.cpp index d1ff56452..4896bb1f0 100644 --- a/tests/src/unit-constructor2.cpp +++ b/tests/src/unit-constructor2.cpp @@ -8,6 +8,9 @@ #include "doctest_compatibility.h" +#include +#include + #include using nlohmann::json; @@ -184,3 +187,193 @@ TEST_CASE("other constructors and destructor") } } } + +TEST_CASE("copy constructor and destructor across value types") +{ + SECTION("copy of a scalar is independent of the source") + { + const json src_null = nullptr; + const json src_bool = true; + const json src_int = -7; + const json src_uint = 7u; + const json src_float = 2.5; + + json c_null(src_null); + json c_bool(src_bool); + json c_int(src_int); + json c_uint(src_uint); + json c_float(src_float); + + CHECK(c_null.is_null()); + CHECK(c_bool == true); + CHECK(c_int == -7); + CHECK(c_uint == 7u); + CHECK(c_float == 2.5); + + c_int = 8; + c_float = 3.5; + CHECK(src_int == -7); + CHECK(src_float == 2.5); + } + + SECTION("copy of an owning value, then mutate the copy") + { + const json src_string = "hello"; + const json src_array = {1, 2, 3}; + const json src_object = {{"a", 1}, {"b", {{"c", 2}}}}; + const json src_binary = json::binary({1, 2, 3}); + + json c_string(src_string); + json c_array(src_array); + json c_object(src_object); + json c_binary(src_binary); + + CHECK(c_string == src_string); + CHECK(c_array == src_array); + CHECK(c_object == src_object); + CHECK(c_binary == src_binary); + + c_string.get_ref() += " world"; + c_array.push_back(4); + c_object["b"]["c"] = 3; + c_object["d"] = 4; + c_binary.get_binary().push_back(4); + + CHECK(src_string == "hello"); + CHECK(src_array.size() == 3); + CHECK(src_object["b"]["c"] == 2); + CHECK_FALSE(src_object.contains("d")); + CHECK(src_binary.get_binary().size() == 3); + + CHECK(c_string == "hello world"); + CHECK(c_array.size() == 4); + CHECK(c_object["b"]["c"] == 3); + CHECK(c_binary.get_binary().size() == 4); + } + + SECTION("copy keeps parent pointers valid (JSON_DIAGNOSTICS)") + { + const json src = {{"a", {1, 2}}, {"b", "x"}}; + json c(src); + c["a"].push_back(3); + CHECK(c["a"].size() == 3); + CHECK(src["a"].size() == 2); + } + + SECTION("vector of mixed types: build, copy, and destroy") + { + { + std::vector v; + for (int i = 0; i < 70; ++i) + { + switch (i % 9) + { + case 0: + v.emplace_back(nullptr); + break; + case 1: + v.emplace_back(i % 2 == 0); + break; + case 2: + v.emplace_back(-i); + break; + case 3: + v.emplace_back(static_cast(i)); + break; + case 4: + v.emplace_back(i * 0.5); + break; + case 5: + v.emplace_back(std::string(static_cast(i), 'x')); + break; + case 6: + v.emplace_back(json::array({i, "s", nullptr})); + break; + case 7: + v.emplace_back(json::object({{"k", i}})); + break; + default: + v.emplace_back(json::binary({1, 2, 3})); + break; + } + } + REQUIRE(v.size() == 70); + + std::vector copy(v); + CHECK(copy == v); + + // mutating the copy leaves the original alone + copy[5] = "changed"; + copy[6].push_back(1); + CHECK(copy != v); + CHECK(v[5] == std::string(5, 'x')); + CHECK(v[6].size() == 3); + + v.erase(v.begin(), v.begin() + 10); + CHECK(v.size() == 60); + } + + // destroying the discarded and default-constructed values + std::vector w(4); + w.emplace_back(json::value_t::discarded); + w.emplace_back(json::value_t::object); + w.emplace_back(json::value_t::array); + w.emplace_back(json::value_t::string); + w.emplace_back(json::value_t::binary); + CHECK(w.size() == 9); + } +} + +TEST_CASE_TEMPLATE("destructor of nested containers", Json, nlohmann::json, nlohmann::ordered_json) +{ + SECTION("empty containers") + { + { + Json j = Json::array(); + } + { + Json j = Json::object(); + } + { + Json j = Json::array({Json::array(), Json::object(), Json::array({Json::array()})}); + } + } + + SECTION("leaves mixed with nested containers") + { + Json j = Json::object(); + j["leaf"] = 1; + j["str"] = "text"; + j["empty_a"] = Json::array(); + j["empty_o"] = Json::object(); + j["arr"] = Json::array({1, "a", Json::array({1, 2}), Json::object({{"k", Json::array({Json::object()})}}), nullptr}); + j["obj"] = Json::object({{"x", Json::object({{"y", Json::array({1, Json::object({{"z", 1}})})}})}, {"w", 2}}); + Json copy = j; + CHECK(copy == j); + j = Json::array({std::move(copy), j, 3, Json::array()}); + } + + SECTION("deep nesting") + { + for (int kind = 0; kind < 3; ++kind) + { + Json j = 1; + for (int i = 0; i < 5000; ++i) + { + switch (kind) + { + case 0: + j = Json::array({std::move(j)}); + break; + case 1: + j = Json::object({{"k", std::move(j)}}); + break; + default: + // siblings next to the nested child + j = (i % 2 == 0) ? Json::array({1, Json::array(), std::move(j)}) : Json::object({{"a", 1}, {"b", Json::object()}, {"c", std::move(j)}}); + break; + } + } + } + } +} diff --git a/tests/src/unit-custom-binary-type.cpp b/tests/src/unit-custom-binary-type.cpp index 6b504bd51..e44a3c88c 100644 --- a/tests/src/unit-custom-binary-type.cpp +++ b/tests/src/unit-custom-binary-type.cpp @@ -8,6 +8,13 @@ #include "doctest_compatibility.h" +// capture whether JSON_DISABLE_ENUM_SERIALIZATION was enabled on the command +// line before including json.hpp, which #undefs it; std::byte is an enum, so +// converting std::vector needs enum serialization +#if defined(JSON_DISABLE_ENUM_SERIALIZATION) && (JSON_DISABLE_ENUM_SERIALIZATION == 1) + #define SKIP_TESTS_FOR_ENUM_SERIALIZATION +#endif + #include #include @@ -85,3 +92,53 @@ TEST_CASE("binary type whose value type is not std::uint8_t") } #endif } + +#ifdef JSON_HAS_CPP_17 +// moved from unit-regression3.cpp, so that file does not instantiate a second +// basic_json specialization (MinGW section limit, see windows.yml) +TEST_CASE("std::vector as binary type") +{ + SECTION("issue #4804: from_cbor incompatible with std::vector as binary_t") + { + const std::vector data = {0x80}; + const auto decoded = byte_binary_json::from_cbor(data); + CHECK((decoded == byte_binary_json::array())); + } + +#ifndef SKIP_TESTS_FOR_ENUM_SERIALIZATION + SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping") + { + // Test that assigning a custom BinaryType directly creates a binary value, not an array + const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}}; + const byte_binary_json j = original; + CHECK(j.is_binary()); + CHECK(!j.is_array()); + + // Test round-tripping: extracting the binary value back as the custom container type + const auto extracted = j.get>(); + CHECK(extracted == original); + + // Test that the default json alias behavior is unchanged: std::vector -> array + const nlohmann::json default_json = std::vector {1, 2, 3}; + CHECK(default_json.is_array()); + CHECK(!default_json.is_binary()); + } + + SECTION("discussion #4209 - custom BinaryType extraction from parsed array") + { + // Test that extracting a custom BinaryType from a parsed JSON array still works + // (not just from a binary-typed node) + const auto j = byte_binary_json::parse("[1,2,3]"); + CHECK(j.is_array()); + CHECK(!j.is_binary()); + + // Extracting as custom BinaryType should work from arrays + const auto extracted = j.get>(); + CHECK(extracted.size() == 3); + CHECK(extracted[0] == std::byte{1}); + CHECK(extracted[1] == std::byte{2}); + CHECK(extracted[2] == std::byte{3}); + } +#endif +} +#endif diff --git a/tests/src/unit-deserialization.cpp b/tests/src/unit-deserialization.cpp index 6222def34..e82685311 100644 --- a/tests/src/unit-deserialization.cpp +++ b/tests/src/unit-deserialization.cpp @@ -35,6 +35,7 @@ using nlohmann::json; #include #include #include +#include #include "sax_event_loggers.hpp" @@ -1250,3 +1251,139 @@ TEST_CASE_TEMPLATE("deserialization of different character types (UTF-32)", T, c CHECK(json::sax_parse(v, &l)); CHECK(l.events.size() == 1); } + +namespace +{ +// records the position and message of a parse error; reporting the error via +// SAX does not throw, so the test also runs with JSON_NOEXCEPTION +struct SaxErrorRecorder : public SaxEventLogger +{ + bool parse_error(std::size_t position, const std::string& /*last_token*/, const json::exception& ex) override + { + error = std::to_string(position) + " " + ex.what(); + return false; + } + + std::string error {}; // NOLINT(readability-redundant-member-init) +}; +} // namespace + +TEST_CASE("parse error positions are the same for contiguous and stream input") +{ + // contiguous input derives the position from the input on error, stream + // input counts it while reading (see lexer::lazy_position); the cases + // cover newlines before the error, ungetting a newline after a number, + // newlines inside a string, a comment, and reads at the end of input + struct test_case + { + std::string input; + std::size_t byte; + std::string message; + }; + const std::vector cases = + { + {"[\n 1,\n 2\n", 12, "[json.exception.parse_error.101] parse error at line 4, column 1: syntax error while parsing array - unexpected end of input; expected ']'"}, + {"[1\n,2,\n]", 8, "[json.exception.parse_error.101] parse error at line 3, column 1: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal"}, + {"[1\n x]", 5, "[json.exception.parse_error.101] parse error at line 2, column 2: syntax error while parsing array - invalid literal; last read: '1 x'; expected ']'"}, + {"1\n2", 3, "[json.exception.parse_error.101] parse error at line 2, column 1: syntax error while parsing value - unexpected number literal; expected end of input"}, + { + "{\"a\":\n\"b\nc\"}", 9, R"([json.exception.parse_error.101] parse error at line 3, column 0: syntax error while parsing value - invalid string: control character U+000A (LF) must be escaped to \u000A or \n; last read: '"b')" + }, + {"/* a\n b */\n x", 13, "[json.exception.parse_error.101] parse error at line 3, column 2: syntax error while parsing value - invalid literal; last read: '/* a b */ x'"}, + {"\n\n", 3, "[json.exception.parse_error.101] parse error at line 3, column 1: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal"}, + {"[\"a\",\n 1e\n]", 10, "[json.exception.parse_error.101] parse error at line 3, column 0: syntax error while parsing value - invalid number; expected '+', '-', or digit after exponent; last read: '1e'"}, + {"[tru\n]", 5, "[json.exception.parse_error.101] parse error at line 2, column 0: syntax error while parsing value - invalid literal; last read: '[tru'"}, + }; + + for (const auto& c : cases) + { + CAPTURE(c.input) + const std::string expected = std::to_string(c.byte) + " " + c.message; + + SaxErrorRecorder string_sax; + CHECK_FALSE(json::sax_parse(c.input, &string_sax, json::input_format_t::json, true, true)); + CHECK(string_sax.error == expected); + + SaxErrorRecorder pointer_sax; + CHECK_FALSE(json::sax_parse(c.input.data(), c.input.data() + c.input.size(), &pointer_sax, json::input_format_t::json, true, true)); + CHECK(pointer_sax.error == expected); + + std::istringstream is(c.input); + SaxErrorRecorder stream_sax; + CHECK_FALSE(json::sax_parse(is, &stream_sax, json::input_format_t::json, true, true)); + CHECK(stream_sax.error == expected); + } +} + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("indentation runs of every length around the 8-byte skip") +{ + // contiguous input skips the spaces after a newline 8 bytes at a time (see + // lexer::skip_indentation), stream input reads them one by one; results, + // byte offsets, and messages must be the same for runs that end anywhere + // relative to a word boundary, including at the end of the input + const auto parse_result = [](const std::string & input, bool contiguous) + { + try + { + if (contiguous) + { + return json::parse(input.data(), input.data() + input.size(), nullptr, true, true).dump(); + } + std::istringstream is(input); + return json::parse(is, nullptr, true, true).dump(); + } + catch (const json::parse_error& e) + { + return std::to_string(e.byte).append(" ").append(e.what()); + } + }; + + // replace every '~' in a pattern by an indentation run + const auto indented = [](const std::string & pattern, const std::string & indent) + { + std::string result; + for (const char c : pattern) + { + if (c == '~') + { + result += indent; + } + else + { + result += c; + } + } + return result; + }; + + const std::vector patterns = + { + "[\n~1,\n~2\n]", + "[\n~1,\n~x]", + "[\n~1,\n~", + "1\n~", + "[\n~\t1,\n~\r\n~2]", + "[\n~/* c */ 1]", + "[\n~1\n~2]", + }; + + for (std::size_t n = 0; n <= 33; ++n) + { + const std::string indent(n, ' '); + for (const auto& pattern : patterns) + { + const std::string input = indented(pattern, indent); + CAPTURE(n) + CAPTURE(input) + const std::string expected = parse_result(input, false); + CHECK(parse_result(input, true) == expected); + CHECK(json::accept(input.data(), input.data() + input.size(), true) == json::accept(input, true)); + } + } + + // and against fixed expectations, in case both paths broke the same way + CHECK(parse_result(indented("[\n~x]", std::string(16, ' ')), true) == "19 [json.exception.parse_error.101] parse error at line 2, column 17: syntax error while parsing value - invalid literal; last read: '[ x'"); + CHECK(parse_result(indented("[\n~1]", std::string(17, ' ')), true) == "[1]"); +} +#endif diff --git a/tests/src/unit-diagnostics.cpp b/tests/src/unit-diagnostics.cpp index 1ada023b7..f791509b4 100644 --- a/tests/src/unit-diagnostics.cpp +++ b/tests/src/unit-diagnostics.cpp @@ -13,9 +13,11 @@ #endif #define JSON_DIAGNOSTICS 1 +#define JSON_TESTS_PRIVATE #include using nlohmann::json; +using nlohmann::ordered_json; #include #include @@ -567,9 +569,9 @@ TEST_CASE("Regression tests for extended diagnostics") } } -TEST_CASE("Better diagnostics past the descent bound of update() and merge_patch()") +TEST_CASE("Better diagnostics past the descent bound of update(), merge_patch() and merge_diff()") { - // Both merge objects nested more than detail::recursion_depth_limit() + // All three handle objects nested more than detail::recursion_depth_limit() // (128) levels deep without recursing; the values they add or replace // there must still know their parents. // The values are built rather than parsed, so that the expected messages @@ -615,4 +617,254 @@ TEST_CASE("Better diagnostics past the descent bound of update() and merge_patch CHECK_THROWS_WITH_AS(p->at("x").at(0), expected_x.c_str(), json::type_error); CHECK_THROWS_WITH_AS(p->at("y").at(0), expected_y.c_str(), json::type_error); } + + SECTION("merge_diff()") + { + // target and patch differ only in their innermost members, so the + // diff of the two is nested as deeply: {"a": ... {"x": null, "y": 2}} + const json diff = json::merge_diff(target, patch); // NOLINT(readability-suspicious-call-argument) + + const json* p = &diff; + for (std::size_t i = 0; i < depth; ++i) + { + p = &p->at("a"); + } + CHECK(p->at("x").is_null()); + CHECK_THROWS_WITH_AS(p->at("y").at(0), expected_y.c_str(), json::type_error); + } +} + +namespace +{ +// check that every child's parent pointer points to the containing value +template +bool parents_consistent(const BasicJsonType& j) +{ + bool ok = true; + if (j.is_array() || j.is_object()) + { + for (const auto& child : j) + { + ok = ok && child.m_parent == &j && parents_consistent(child); + } + } + return ok; +} +} // namespace + +TEST_CASE_TEMPLATE("Parent pointers stay consistent", J, json, ordered_json) +{ + SECTION("parse, copy, move, assign, swap") + { + J j = J::parse(R"({"a":[1,{"b":[2,3]},[4]],"c":{"d":{"e":[]}}})"); + CHECK(parents_consistent(j)); + + J copy(j); + CHECK(parents_consistent(copy)); + + J moved(std::move(copy)); + CHECK(parents_consistent(moved)); + + J assigned; + assigned = j; + CHECK(parents_consistent(assigned)); + + J other = {1, 2, {{"k", J::array({1, 2})}}}; + other.swap(assigned); + CHECK(parents_consistent(other)); + CHECK(parents_consistent(assigned)); + } + + SECTION("push_back growth") + { + J arr = J::array(); + for (int i = 0; i < 500; ++i) + { + arr.push_back(J::array({i})); + } + CHECK(parents_consistent(arr)); + + J obj = J::object(); + for (int i = 0; i < 500; ++i) + { + obj.push_back({std::to_string(i), J::array({i})}); + } + CHECK(parents_consistent(obj)); + } + + SECTION("emplace_back and emplace growth") + { + J arr = J::array(); + J obj = J::object(); + for (int i = 0; i < 500; ++i) + { + arr.emplace_back(J::array({i})); + obj.emplace(std::to_string(i), J::array({i})); + } + CHECK(parents_consistent(arr)); + CHECK(parents_consistent(obj)); + } + + SECTION("operator[] growth and at()") + { + J obj = J::object(); + for (int i = 0; i < 500; ++i) + { + obj[std::to_string(i)] = J::array({i}); + CHECK(obj[std::to_string(i / 2)].m_parent == &obj); + } + CHECK(parents_consistent(obj)); + + J arr = J::array(); + for (int i = 0; i < 500; ++i) + { + arr[static_cast(i)] = J::object({{"x", i}}); + } + for (size_t i = 0; i < 500; ++i) + { + CHECK(arr.at(i).m_parent == &arr); + } + CHECK(obj.at("3").m_parent == &obj); + CHECK(parents_consistent(arr)); + CHECK(parents_consistent(obj)); + } +} + +TEST_CASE_TEMPLATE("update() and merge_patch() keep parent pointers and diagnostic paths", J, json, ordered_json) +{ + // enough keys to make the vector behind ordered_json reallocate several times + const auto make_object = [](int first, int last) + { + J o = J::object(); + for (int i = first; i < last; ++i) + { + o[std::to_string(i)] = J::object({{"x", J::array({i})}}); + } + return o; + }; + + // const access, so operator[] cannot repair the parent pointers + const auto check_path = [](const J & j, const std::string & key) + { + const std::string expected = "[json.exception.type_error.304] (/" + key + "/x) cannot use at() with array"; + CHECK_THROWS_WITH_AS(j[key]["x"].at("y"), expected.c_str(), json::type_error); + }; + + SECTION("update(const_reference)") + { + J j = make_object(0, 200); + j.update(make_object(100, 400)); + CHECK(j.size() == 400); + CHECK(parents_consistent(j)); + check_path(j, "0"); + check_path(j, "150"); + check_path(j, "399"); + } + + SECTION("update(const_reference, merge_objects)") + { + J j = make_object(0, 200); + j["50"]["y"] = 1; + J other = make_object(100, 400); + other["50"] = J::object({{"z", J::array({1})}}); + j.update(other, true); + CHECK(j.size() == 400); + CHECK(j["50"].size() == 3); + CHECK(parents_consistent(j)); + check_path(j, "0"); + check_path(j, "399"); + } + + SECTION("update(first, last)") + { + J j = make_object(0, 200); + J other = make_object(100, 400); + j.update(other.begin(), other.end()); + CHECK(j.size() == 400); + CHECK(parents_consistent(j)); + check_path(j, "0"); + check_path(j, "399"); + + J k = make_object(0, 200); + k.update(other.begin(), other.end(), true); + CHECK(k.size() == 400); + CHECK(parents_consistent(k)); + check_path(k, "399"); + } + + SECTION("merge_patch()") + { + J j = make_object(0, 200); + J patch = make_object(100, 400); + patch["0"] = nullptr; + j.merge_patch(patch); + CHECK(j.size() == 399); + CHECK_FALSE(j.contains("0")); + CHECK(parents_consistent(j)); + check_path(j, "1"); + check_path(j, "399"); + } +} + +namespace +{ +// an object type that, like a sorted flat map, moves its members when a key is +// added even if its capacity does not change; it has a capacity() member, but +// is neither node-based nor nlohmann::ordered_map, so the library must not +// trust its capacity to tell whether the members moved +template, + class Allocator = std::allocator>> + struct relocating_map : nlohmann::ordered_map +{ + using base_type = nlohmann::ordered_map; + using typename base_type::iterator; + using typename base_type::Container; + using base_type::base_type; + + template + std::pair emplace(Args&& ... args) + { + auto result = base_type::emplace(std::forward(args)...); + const auto index = result.first - this->begin(); + if (result.second) + { + // move every member to a new buffer of the same capacity + Container relocated; + relocated.reserve(this->capacity()); + for (auto& member : *this) + { + relocated.emplace_back(member.first, std::move(member.second)); + } + this->Container::swap(relocated); + } + return {this->begin() + index, result.second}; + } +}; + +using relocating_json = nlohmann::basic_json; +} // namespace + +TEST_CASE("Parent pointers stay consistent for object types not known to keep their members in place") +{ + relocating_json obj = relocating_json::object(); + for (int i = 0; i < 100; ++i) + { + obj[std::to_string(i)] = relocating_json::array({i}); + obj.emplace(std::to_string(i + 1000), relocating_json::array({i})); + } + CHECK(parents_consistent(obj)); + + obj.erase("0"); + CHECK(parents_consistent(obj)); + + relocating_json other = relocating_json::object(); + for (int i = 0; i < 100; ++i) + { + other[std::to_string(i + 2000)] = relocating_json::array({i}); + } + obj.update(other); + CHECK(parents_consistent(obj)); + + obj["1"].push_back(42); + CHECK_THROWS_WITH_AS(obj["1"].at(5), "[json.exception.out_of_range.401] (/1) array index 5 is out of range", relocating_json::out_of_range&); } diff --git a/tests/src/unit-iterators1.cpp b/tests/src/unit-iterators1.cpp index b7b0aa9eb..629c6abd7 100644 --- a/tests/src/unit-iterators1.cpp +++ b/tests/src/unit-iterators1.cpp @@ -10,6 +10,8 @@ #define JSON_TESTS_PRIVATE #include +#include +#include using nlohmann::json; TEST_CASE("iterators 1") @@ -1628,3 +1630,108 @@ TEST_CASE("iterators 1") } } } + +TEST_CASE_TEMPLATE("reverse iteration and member access", Json, nlohmann::json, nlohmann::ordered_json) +{ + SECTION("array: -- and rbegin/rend") + { + Json j = {1, 2, 3, 4}; + + auto it = j.end(); + std::vector seen; + while (it != j.begin()) + { + --it; + seen.push_back(it->template get()); + } + CHECK(seen == std::vector {4, 3, 2, 1}); + + auto it2 = j.end(); + auto old = it2--; + CHECK(old == j.end()); + CHECK(*it2 == 4); + + seen.clear(); + for (auto r = j.rbegin(); r != j.rend(); ++r) + { + seen.push_back(r->template get()); + } + CHECK(seen == std::vector {4, 3, 2, 1}); + + seen.clear(); + for (auto r = j.crbegin(); r != j.crend(); ++r) + { + seen.push_back(*r); + } + CHECK(seen == std::vector {4, 3, 2, 1}); + } + + SECTION("object: -- and rbegin/rend") + { + Json j = {{"a", 1}, {"b", 2}, {"c", 3}}; + // order of keys depends on the object type; use the forward order as reference + std::vector fwd; + for (auto it = j.begin(); it != j.end(); ++it) + { + fwd.push_back(it.key()); + } + std::vector rev(fwd.rbegin(), fwd.rend()); + + std::vector seen; + auto it = j.end(); + while (it != j.begin()) + { + --it; + seen.push_back(it.key()); + } + CHECK(seen == rev); + + auto it2 = j.end(); + auto old = it2--; + CHECK(old == j.end()); + CHECK(it2.key() == rev.front()); + + seen.clear(); + for (auto r = j.rbegin(); r != j.rend(); ++r) + { + seen.push_back(r.key()); + } + CHECK(seen == rev); + } + + SECTION("object: operator-> reads and writes the mapped value") + { + Json j = {{"a", 1}, {"s", "str"}}; + auto it = j.find("s"); + REQUIRE(it != j.end()); + CHECK(it->is_string()); + CHECK(it->size() == 1); + *it = Json::array({1, 2}); + CHECK(it->is_array()); + it->push_back(3); + CHECK(j["s"].size() == 3); + + const Json& cj = j; + auto cit = cj.find("a"); + REQUIRE(cit != cj.end()); + CHECK(cit->is_number_integer()); + CHECK(cit->template get() == 1); + } + + SECTION("primitives: -- and ->") + { + Json j = 42; + auto it = j.end(); + --it; + CHECK(it == j.begin()); + CHECK(it->template get() == 42); + } + + SECTION("comparing iterators of different containers throws") + { + Json a = {1, 2}; + Json b = {1, 2}; + CHECK_THROWS_WITH_AS(a.begin() == b.begin(), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", nlohmann::json::invalid_iterator); + CHECK_THROWS_WITH_AS(a.begin() < b.begin(), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", nlohmann::json::invalid_iterator); + } +} diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index 9e83edbcf..67ffee6dd 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -2066,3 +2066,40 @@ TEST_CASE("JSON patch - every operation on ordered_json") } } } + +TEST_CASE_TEMPLATE("JSON patch - diff round trip on hand-made documents", Json, nlohmann::json, nlohmann::ordered_json) +{ + const auto roundtrip = [](const Json & a, const Json & b) + { + CAPTURE(a.dump()) + CAPTURE(b.dump()) + const Json p = Json::diff(a, b); + CHECK(a.patch(p) == b); + CHECK(Json::diff(b, b).empty()); + }; + + SECTION("reordered keys") + { + const Json a = Json::parse(R"({"x":1,"y":2,"z":{"p":1,"q":2}})"); + const Json b = Json::parse(R"({"z":{"q":2,"p":1},"y":2,"x":1})"); + roundtrip(a, b); + roundtrip(b, a); + } + + SECTION("nested add, remove and replace") + { + const Json a = Json::parse(R"({"a":[1,2,{"k":"v","gone":true}],"b":{"c":{"d":1}},"e":"s"})"); + const Json b = Json::parse(R"({"a":[1,3,{"k":"w","new":[]}],"b":{"c":{"d":[1,2]}},"f":null})"); + roundtrip(a, b); + roundtrip(b, a); + } + + SECTION("arrays of different length and type changes") + { + roundtrip(Json::parse("[1,2,3,4,5]"), Json::parse("[1,3]")); + roundtrip(Json::parse("[1]"), Json::parse("[1,[2],{\"a\":3}]")); + roundtrip(Json::parse("[]"), Json::parse("{}")); + roundtrip(Json::parse("{\"a\":[]}"), Json::parse("{\"a\":{}}")); + roundtrip(Json(1), Json::parse("[1]")); + } +} diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index 658a7fa8d..39c4f790c 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -23,8 +23,14 @@ using nlohmann::json; using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif +#include +#include +#include #include +#include #include +#include +#include TEST_CASE("JSON pointers") { @@ -1071,3 +1077,242 @@ TEST_CASE("flatten of structured values") R"({"/z":"s","/a~1b/~0":1,"/a~1b/k":null,"/a/0":1,"/a/1/c":null,"/a/1/b":null,"/a/2/0/x~0~1/0":true,"/a/2/0/x~0~1/1":null,"/a/2/0/w":2})"); } } + +TEST_CASE("flatten of ordered_json") +{ + using nlohmann::ordered_json; + + SECTION("keeps the document order of the leaves") + { + const ordered_json j = ordered_json::parse(R"({ + "z": 1, "a": {"y": [true, null, {}], "b": "x", "": []}, + "m~n": 2.5, "o/p": {"k": {}}, "0": [[1], [2, 3]] + })"); + + const ordered_json flat = j.flatten(); + std::vector keys; + for (const auto& el : flat.items()) + { + keys.push_back(el.key()); + } + const std::vector expected = + { + "/z", "/a/y/0", "/a/y/1", "/a/y/2", "/a/b", "/a/", "/m~0n", "/o~1p/k", "/0/0/0", "/0/1/0", "/0/1/1" + }; + CHECK(keys == expected); + CHECK(flat.size() == expected.size()); + CHECK(flat["/m~0n"] == 2.5); + CHECK(flat["/a/y/2"].is_null()); + } + + SECTION("round-trips and agrees with json") + { + ordered_json j; + for (int i = 0; i < 200; ++i) + { + j["k" + std::to_string(i)] = {{"v", i}, {"l", {i, i + 1, "s"}}}; + } + const ordered_json flat = j.flatten(); + CHECK(flat.size() == 800); + CHECK(flat.unflatten() == j); + + // same entries as for a sorted object + const nlohmann::json plain = nlohmann::json::parse(j.dump()); + CHECK(nlohmann::json::parse(flat.dump()) == plain.flatten()); + } + + SECTION("primitive and empty root") + { + CHECK(ordered_json(1).flatten() == ordered_json({{"", 1}})); + CHECK(ordered_json::array().flatten() == ordered_json({{"", nullptr}})); + } +} +namespace +{ +// reference implementation of the array index rules (RFC 6901, Sect. 4) based on strtoull +enum class index_class +{ + ok, + leading_zero, + not_a_number, + unresolved, + exceeds_size_type +}; + +index_class classify_index(const std::string& s, std::size_t& idx) +{ + if (s.size() > 1 && s[0] == '0') + { + return index_class::leading_zero; + } + if (!s.empty() && s != "-" && !(s[0] >= '0' && s[0] <= '9')) + { + return index_class::not_a_number; + } + char* end = nullptr; + errno = 0; + const unsigned long long res = std::strtoull(s.c_str(), &end, 10); // NOLINT(runtime/int) + if (end == s.c_str() || errno == ERANGE || static_cast(end - s.c_str()) != s.size()) + { + return index_class::unresolved; + } + if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) + { + return index_class::exceeds_size_type; + } + idx = static_cast(res); + return index_class::ok; +} + +// the observable behavior of json_pointer::at() and contains() for an array index token +void check_index_token(const std::string& token) +{ + const json array = {10, 20, 30}; + const json::json_pointer ptr("/" + token); + + std::size_t idx = 0; + const auto cls = classify_index(token, idx); + const bool exists = cls == index_class::ok && idx < array.size(); + + CAPTURE(token) + CHECK(array.contains(ptr) == (exists && token != "-")); + + if (token == "-") + { + CHECK_THROWS_AS(array.at(ptr), json::out_of_range&); + return; + } + + switch (cls) + { + case index_class::leading_zero: + CHECK_THROWS_AS(array.at(ptr), json::parse_error&); + break; + case index_class::not_a_number: + CHECK_THROWS_AS(array.at(ptr), json::parse_error&); + break; + case index_class::unresolved: + case index_class::exceeds_size_type: + CHECK_THROWS_AS(array.at(ptr), json::out_of_range&); + break; + case index_class::ok: + default: + if (exists) + { + CHECK(array.at(ptr) == array[idx]); + } + else + { + CHECK_THROWS_AS(array.at(ptr), json::out_of_range&); + } + break; + } +} +} // namespace + +TEST_CASE("JSON pointer array index tokens match strtoull semantics") +{ + SECTION("boundary values") + { + const std::string max_digits = std::to_string((std::numeric_limits::max)()); + const std::vector tokens = + { + "", "-", "0", "1", "2", "3", "00", "01", "007", "+1", "-1", " 1", "1 ", "1a", "a1", "0x1", "1e1", "1.0", + "999999999", "1000000000", "99999999999999999", "999999999999999999", "1000000000000000000", + "9999999999999999999", "18446744073709551614", max_digits, "18446744073709551616", + "99999999999999999999999", "000000000000000000000" + }; + for (const auto& token : tokens) + { + check_index_token(token); + } + } + + SECTION("random tokens") + { + std::mt19937 gen(42); + const std::string alphabet = "0123456789012345678901234567890123456789-+ ex"; + for (int i = 0; i < 5000; ++i) + { + std::string token(static_cast(gen() % 23), '0'); + for (auto& c : token) + { + c = alphabet[gen() % alphabet.size()]; + } + check_index_token(token); + } + } +} + +TEST_CASE("JSON pointer operations on pointers with escaped tokens") +{ + const std::vector strings = + { + "", "/", "//", "/a", "/a/b", "/a~1b", "/~0", "/~01", "/~10", "/a~0~1/b/", "/0/1/2", "/ ", "/a/", "/~1/~0" + }; + + for (const auto& str : strings) + { + CAPTURE(str) + const json::json_pointer ptr(str); + CHECK(ptr.to_string() == str); + CHECK(ptr / "x" / "y" / std::size_t{3} == json::json_pointer(str + "/x/y/3")); + CHECK(ptr.parent_pointer() == (str.empty() ? ptr : json::json_pointer(str.substr(0, str.rfind('/'))))); + + // comparison with a string, in place, and with every other pointer +#if !defined(JSON_TEST_DEPRECATED_FUNCTIONS_DELETED) + for (const auto& other : strings) + { + CAPTURE(other) + CHECK((ptr == other) == (str == other)); + CHECK((other == ptr) == (str == other)); + CHECK((ptr != other) == (str != other)); + } + CHECK_THROWS_AS(ptr == "x", json::parse_error&); + CHECK_THROWS_AS(ptr == "/~", json::parse_error&); + CHECK_THROWS_AS(ptr == "/b/~2", json::parse_error&); +#endif + } +} + +TEST_CASE("JSON pointer lookups") +{ + json j = {{"a", {{"b", {1, 2, 3}}}}, {"", 7}, {"~/", {{"x", nullptr}}}}; + const json& cj = j; + + CHECK(cj[json::json_pointer("/a/b/2")] == 3); + CHECK(cj.at(json::json_pointer("/a/b/1")) == 2); + CHECK(cj.at(json::json_pointer("/")) == 7); + CHECK(cj.contains(json::json_pointer("/~0~1/x"))); + CHECK(!cj.contains(json::json_pointer("/a/c"))); + CHECK(cj.value(json::json_pointer("/a/b/0"), 5) == 1); + CHECK(cj.value(json::json_pointer("/a/b/3"), 5) == 5); + CHECK(cj.value(json::json_pointer("/a/zz"), 5) == 5); + + CHECK_THROWS_AS(cj.at(json::json_pointer("/a/c")), json::out_of_range&); + CHECK_THROWS_AS(cj.at(json::json_pointer("/a/b/3")), json::out_of_range&); + + // non-const access reuses existing entries and creates missing ones + j[json::json_pointer("/a/b/1")] = 20; + j[json::json_pointer("/n/m")] = 1; + CHECK(j["a"]["b"] == json({1, 20, 3})); + CHECK(j["n"]["m"] == 1); +} + +TEST_CASE("JSON patch with moves and copies") +{ + json doc = {{"a", {{"b", {1, 2, 3}}}}, {"c", {{"d", "x"}}}}; + + CHECK(doc.patch(json::parse(R"([{"op":"move","from":"/a/b","path":"/c/e"}])")) == + json::parse(R"({"a":{},"c":{"d":"x","e":[1,2,3]}})")); + CHECK(doc.patch(json::parse(R"([{"op":"copy","from":"/a/b","path":"/c/e"}])")) == + json::parse(R"({"a":{"b":[1,2,3]},"c":{"d":"x","e":[1,2,3]}})")); + CHECK(doc.patch(json::parse(R"([{"op":"move","from":"/a/b/0","path":"/a/b/2"}])")) == + json::parse(R"({"a":{"b":[2,3,1]},"c":{"d":"x"}})")); + + // a failing move leaves a copy-patched document untouched + CHECK_THROWS_AS(doc.patch(json::parse(R"([{"op":"move","from":"","path":""}])")), json::out_of_range&); + json inplace = doc; + CHECK_THROWS_AS(inplace.patch_inplace(json::parse(R"([{"op":"move","from":"","path":""}])")), json::out_of_range&); + CHECK(inplace == doc); +} diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index 4865e5687..b665b6531 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -260,10 +260,11 @@ struct LocaleSwitchingSax final: public nlohmann::json_sax TEST_CASE("locale changes between lexer construction and number conversion (#5198)") { - // The numbers are chosen so that the conversion also takes the strtod - // fallback, which honors the locale that is current at conversion time: - // too many significant digits for Clinger's fast path, an underflow that - // std::from_chars rejects, and a plain value. + // float and double are converted without the locale. A long double that + // is not binary64 can take the strtold fallback, which honors the locale + // that is current at conversion time. The numbers are chosen so that it + // does: too many significant digits for Clinger's fast path, an underflow + // that std::from_chars rejects, and a plain value. const std::vector numbers = {"3.14159265358979323846", "1.5e-400", "12.34", "-0.000123456789012345678"}; std::string text = "["; for (const auto& n : numbers) @@ -327,7 +328,8 @@ TEST_CASE("locale changes between lexer construction and number conversion (#519 } } - // a long double goes through std::strtold unless std::from_chars supports it + // a long double goes through std::strtold unless it is binary64 or + // std::from_chars supports it { bool switched = false; const auto cb = [&](int /*depth*/, long_double_json::parse_event_t event, long_double_json& /*parsed*/) noexcept @@ -353,8 +355,15 @@ TEST_CASE("locale with a multi-byte decimal point") { // Some locales use a decimal point that is not a single character, e.g. // U+066B ARABIC DECIMAL SEPARATOR (two bytes in UTF-8). It cannot be - // substituted in place for '.', so the strtod fallback stops early. The - // conversion must still terminate rather than retry forever. + // substituted in place for '.', so the strtold fallback (only for long + // double formats other than binary64) converts a copy of the token with + // the whole decimal point instead (#5660). The values must be those of the + // "C" locale. + using long_double_json = nlohmann::basic_json; + const char* const long_double_numbers = "[3.14159265358979323846, 1.5e-400, -0.000123456789012345678]"; + REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr); + const long_double_json expected_long_double = long_double_json::parse(long_double_numbers); + const std::array names = {{"ar_EG.UTF-8", "ar_SA.UTF-8", "fa_IR.UTF-8", "ps_AF.UTF-8", "ar_EG", "fa_IR"}}; bool tested = false; for (const char* name : names) @@ -372,12 +381,20 @@ TEST_CASE("locale with a multi-byte decimal point") tested = true; // too many significant digits for Clinger's fast path, and an underflow - // that std::from_chars rejects: both reach the strtod fallback + // that std::from_chars rejects: double does not depend on the locale json j; CHECK_NOTHROW(j = json::parse("[3.14159265358979323846, 1.5e-400, -0.000123456789012345678]")); CHECK(j.is_array()); + CHECK(j[0] == 3.14159265358979323846); + CHECK(j[1] == 0.0); + CHECK(j[2] == -0.000123456789012345678); CHECK(json::accept("3.14159265358979323846")); + // a long double that reaches the strtold fallback is not truncated + long_double_json ld; + CHECK_NOTHROW(ld = long_double_json::parse(long_double_numbers)); + CHECK(ld == expected_long_double); + // a value the locale-independent paths convert is not affected CHECK(json::parse("12.5") == 12.5); } diff --git a/tests/src/unit-merge_diff.cpp b/tests/src/unit-merge_diff.cpp new file mode 100644 index 000000000..77fa58768 --- /dev/null +++ b/tests/src/unit-merge_diff.cpp @@ -0,0 +1,1024 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; +using nlohmann::ordered_json; +#ifdef JSON_TEST_NO_GLOBAL_UDLS + using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) +#endif + +#include +#include + +namespace +{ +// merge_diff() recursing once per nesting level, without the bound on its +// descent; only usable on values nested a few hundred levels deep +template +BasicJsonType reference_merge_diff(const BasicJsonType& source, const BasicJsonType& target) +{ + if (!source.is_object() || !target.is_object()) + { + return target; + } + + BasicJsonType result = BasicJsonType::object(); + for (auto it = source.begin(); it != source.end(); ++it) + { + const auto itf = target.find(it.key()); + if (itf == target.end()) + { + result[it.key()] = nullptr; + } + else if (it.value().is_object() && itf.value().is_object()) + { + auto diff = reference_merge_diff(it.value(), itf.value()); + if (!diff.empty()) + { + result[it.key()] = std::move(diff); + } + } + else if (it.value() != itf.value()) + { + result[it.key()] = itf.value(); + } + } + for (auto it = target.begin(); it != target.end(); ++it) + { + if (source.find(it.key()) == source.end()) + { + result[it.key()] = it.value(); + } + } + return result; +} + +// objects nested `depth` levels deep under the key "a", with members before +// and after it that differ by `variant` on the way down, so that diffing two +// variants removes, adds and replaces members on many levels, and diffs nested +// objects that are equal, that differ, or that are not objects in one of the +// variants; only variant 2 has members that are null +template +BasicJsonType nested_objects(const std::size_t depth, const int variant) +{ + BasicJsonType value = BasicJsonType::object({{"v", variant}}); + for (std::size_t i = 0; i < depth; ++i) + { + BasicJsonType object = BasicJsonType::object(); + if ((i + static_cast(variant)) % 3 == 0) + { + object["s" + std::to_string(variant)] = i; + } + object["c"] = i % 4 == 0 ? variant : 0; + if (i % 5 == 0) + { + if (variant == 0) + { + object["o"] = BasicJsonType::object({{"p", i}}); + } + else if (variant == 1) + { + object["o"] = BasicJsonType::array({i}); + } + else + { + object["o"] = nullptr; + } + } + if (variant == 2 && i % 4 == 2) + { + object["n"] = nullptr; + } + object["a"] = std::move(value); + if (i % 6 == 0) + { + object["d"] = BasicJsonType::object({{"e", variant == 1 ? 1 : 0}}); + } + if (variant == 1 && i % 7 == 0) + { + object["z"] = "z"; + } + value = std::move(object); + } + return value; +} +} // namespace + +TEST_CASE("JSON Merge Patch diff") +{ + SECTION("examples from RFC 7396") + { + SECTION("Section 1") + { + json document = R"({ + "a": "b", + "c": { + "d": "e", + "f": "g" + } + })"_json; + + json expected = R"({ + "a": "z", + "c": { + "d": "e" + } + })"_json; + + json const expected_patch = R"({ + "a": "z", + "c": { + "f": null + } + })"_json; + + auto patch = json::merge_diff(document, expected); + CHECK(patch == expected_patch); + + document.merge_patch(patch); + CHECK(document == expected); + } + + SECTION("Section 3") + { + json document = R"({ + "title": "Goodbye!", + "author": { + "givenName": "John", + "familyName": "Doe" + }, + "tags": [ + "example", + "sample" + ], + "content": "This will be unchanged" + })"_json; + + json expected = R"({ + "title": "Hello!", + "author": { + "givenName": "John" + }, + "tags": [ + "example" + ], + "content": "This will be unchanged", + "phoneNumber": "+01-123-456-7890" + })"_json; + + json const expected_patch = R"({ + "title": "Hello!", + "phoneNumber": "+01-123-456-7890", + "author": { + "familyName": null + }, + "tags": [ + "example" + ] + })"_json; + + auto patch = json::merge_diff(document, expected); + CHECK(patch == expected_patch); + + document.merge_patch(patch); + CHECK(document == expected); + } + + SECTION("Appendix A") + { + SECTION("Example 1") + { + json original = R"({"a":"b"})"_json; + json result = R"({"a":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":"c"})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 2") + { + json original = R"({"a":"b"})"_json; + json result = R"({"a":"b", "b":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"b":"c"})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 3") + { + json original = R"({"a":"b"})"_json; + json result = R"({})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 4") + { + json original = R"({"a":"b","b":"c"})"_json; + json result = R"({"b":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 5") + { + json original = R"({"a":["b"]})"_json; + json result = R"({"a":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":"c"})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 6") + { + json original = R"({"a":"c"})"_json; + json result = R"({"a":["b"]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":["b"]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 7") + { + json original = R"({"a":{"b": "c"}})"_json; + json result = R"({"a": {"b": "d"}})"_json; + + auto patch = json::merge_diff(original, result); + // differs from the RFC 7396, "c": null is not in the patch + // because neither the source nor the target have the "c" key + CHECK(patch == R"({"a":{"b":"d"}})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 8") + { + json original = R"({"a":[{"b":"c"}]})"_json; + json result = R"({"a":[1]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":[1]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 9") + { + json original = R"(["a","b"])"_json; + json result = R"(["c","d"])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(["c","d"])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 10") + { + json original = R"({"a":"b"})"_json; + json result = R"(["c"])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(["c"])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 11") + { + json original = R"({"a":"foo"})"_json; + json result = R"(null)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(null)"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 12") + { + json original = R"({"a":"foo"})"_json; + json result = R"("bar")"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"("bar")"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 13") + { + json original = R"({"e":null})"_json; + json result = R"({"e":null,"a":1})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":1})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 14") + { + json original = R"([1,2])"_json; + json result = R"({"a":"b"})"_json; + + auto patch = json::merge_diff(original, result); + // differs from the RFC 7396, "c": null is not in the patch + // because neither the source nor the target have the "c" key + CHECK(patch == R"({"a":"b"})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("Example 15") + { + json original = R"({})"_json; + json result = R"({"a":{"bb":{}}})"_json; + + auto patch = json::merge_diff(original, result); + // differs from the RFC 7396, "ccc": null is not in the patch + // because neither the source nor the target have the "ccc" key + CHECK(patch == R"({"a":{"bb":{}}})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + } + } + + SECTION("null values") + { + SECTION("object with null value to object") + { + json original = R"({"a":null})"_json; + json result = R"({"a":{"b":"c"}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":"c"}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{"b":"c"}})"_json); + } + + SECTION("object to object with null value") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("primitive to object with null value") + { + json original = R"({"a":1})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("nested primitive to object with null value") + { + json original = R"({"a":{"b":1}})"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("array value to object with null value") + { + json original = R"({"a":[1,2]})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("nested object to object with null value") + { + json original = R"({"a":{"b":{"c":"d"}}})"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("nested primitive to object with null value with sibling") + { + json original = R"({"x":"y","a":{"b":"c"}})"_json; + json result = R"({"x":"y","a":{"b":null, "c":"d"}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null,"c":"d"}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"x":"y","a":{"c":"d"}})"_json); + } + + SECTION("empty object to object with null value") + { + json original = R"({})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("null to object with null value") + { + json original = R"(null)"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("array to object with null value") + { + json original = R"([])"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == R"({})"_json); + } + + SECTION("empty object to nested null value") + { + json original = R"({})"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("object to nested null value") + { + json original = R"({"a": 1})"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("primitive to nested null value") + { + json original = R"(1)"_json; + json result = R"({"a":{"b":null}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{}})"_json); + } + + SECTION("object with null value unchanged") + { + json original = R"({"a":null})"_json; + json result = R"({"a":null})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object with null value to empty object") + { + json original = R"({"a":null})"_json; + json result = R"({})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":null})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object with null value to primitive") + { + json original = R"({"a":null})"_json; + json result = R"({"a":1})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":1})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("array to array with null value") + { + json original = R"({"a":[1]})"_json; + json result = R"({"a":[null]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":[null]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("empty object to array with null value") + { + json original = R"({})"_json; + json result = R"({"a":[null]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":[null]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("doubly nested primitive to null value") + { + json original = R"({"a":{"b":{"c":1}}})"_json; + json result = R"({"a":{"b":{"c":null}}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":{"c":null}}})"_json); + + original.merge_patch(patch); + CHECK(original == R"({"a":{"b":{}}})"_json); + } + } + + SECTION("no change") + { + SECTION("object") + { + json original = R"({"a":"b","b":"c"})"_json; + json result = R"({"a":"b","b":"c"})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("empty object") + { + json original = R"({})"_json; + json result = R"({})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("array") + { + json original = R"([1,2,3])"_json; + json result = R"([1,2,3])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(!patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("null") + { + json original = R"(null)"_json; + json result = R"(null)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("string") + { + json original = R"("ab")"_json; + json result = R"("ab")"_json; + + auto patch = json::merge_diff(original, result); + CHECK(!patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("number") + { + json original = R"(42)"_json; + json result = R"(42)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(!patch.empty()); + original.merge_patch(patch); + CHECK(original == result); + } + } + + SECTION("primitives") + { + SECTION("string") + { + json original = R"("a")"_json; + json result = R"("b")"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"("b")"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("number") + { + json original = R"(1)"_json; + json result = R"(2)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(2)"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("boolean") + { + json original = R"(false)"_json; + json result = R"(true)"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"(true)"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to primitive") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":1})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":1})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("primitive to object") + { + json original = R"({"a":1})"_json; + json result = R"({"a":{"b":"c"}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":"c"}})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to empty string") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":""})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":""})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to zero") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":0})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":0})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to empty object") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":{}})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":{"b":null}})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + } + + SECTION("arrays") + { + SECTION("array to array") + { + json original = R"([1,2,3])"_json; + json result = R"([1,2,4])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"([1,2,4])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("array to empty array") + { + json original = R"([1,2,3])"_json; + json result = R"([])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"([])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("empty array to array") + { + json original = R"([])"_json; + json result = R"([1,2,3])"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"([1,2,3])"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("object to empty array") + { + json original = R"({"a":{"b":"c"}})"_json; + json result = R"({"a":[]})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":[]})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("same keys") + { + json original = R"(["a","b"])"_json; + json result = R"({"a":1,"b":2})"_json; + + auto patch = json::merge_diff(original, result); + CHECK(patch == R"({"a":1,"b":2})"_json); + + original.merge_patch(patch); + CHECK(original == result); + } + } + + SECTION("ordered_json") + { + SECTION("changed value keeps member order") + { + auto original = ordered_json::parse(R"({"c":1,"b":2,"a":3})"); + auto result = ordered_json::parse(R"({"c":1,"b":5,"a":3})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"b":5})"); + + original.merge_patch(patch); + CHECK(original == result); + CHECK(original.dump() == R"({"c":1,"b":5,"a":3})"); + } + + SECTION("added members follow target order") + { + auto original = ordered_json::parse(R"({"b":1})"); + auto result = ordered_json::parse(R"({"b":1,"z":2,"a":3})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"z":2,"a":3})"); + + original.merge_patch(patch); + CHECK(original == result); + CHECK(original.dump() == R"({"b":1,"z":2,"a":3})"); + } + + SECTION("removed members follow source order") + { + auto original = ordered_json::parse(R"({"c":1,"b":2,"a":3})"); + auto result = ordered_json::parse(R"({"b":2})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"c":null,"a":null})"); + + original.merge_patch(patch); + CHECK(original == result); + } + + SECTION("nested objects keep member order") + { + auto original = ordered_json::parse(R"({"z":{"y":1,"x":2},"a":0})"); + auto result = ordered_json::parse(R"({"z":{"y":1,"x":3,"w":4},"a":0})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"z":{"x":3,"w":4}})"); + + original.merge_patch(patch); + CHECK(original == result); + CHECK(original.dump() == R"({"z":{"y":1,"x":3,"w":4},"a":0})"); + } + + SECTION("reordered members cannot be expressed") + { + auto original = ordered_json::parse(R"({"a":1,"b":2})"); + auto result = ordered_json::parse(R"({"b":2,"a":1})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.empty()); + + original.merge_patch(patch); + CHECK(original != result); + CHECK(original.dump() == R"({"a":1,"b":2})"); + } + + SECTION("member added before existing ones is appended") + { + auto original = ordered_json::parse(R"({"b":1})"); + auto result = ordered_json::parse(R"({"a":0,"b":1})"); + + auto patch = ordered_json::merge_diff(original, result); + CHECK(patch.dump() == R"({"a":0})"); + + original.merge_patch(patch); + CHECK(original != result); + CHECK(original.dump() == R"({"b":1,"a":0})"); + } + } +} + +TEST_CASE("JSON Merge Patch diff of deeply nested values") +{ + SECTION("the patch is the same past the descent bound") + { + // depths on either side of the nesting depth up to which merge_diff() + // recurses (detail::recursion_depth_limit(), 128); not every depth up + // to 300, to keep the test fast under Valgrind + std::vector depths; + for (std::size_t depth = 0; depth <= 16; ++depth) + { + depths.push_back(depth); + } + for (std::size_t depth = 120; depth <= 136; ++depth) + { + depths.push_back(depth); + } + depths.push_back(300); + + for (const auto depth : depths) + { + CAPTURE(depth) + for (int from = 0; from < 3; ++from) + { + for (int to = 0; to < 3; ++to) + { + CAPTURE(from) + CAPTURE(to) + const auto source = nested_objects(depth, from); + const auto target = nested_objects(depth, to); + const auto patch = json::merge_diff(source, target); + CHECK(patch == reference_merge_diff(source, target)); + CHECK(patch.empty() == (from == to)); + + // a merge patch cannot set a member to null, so it only + // reproduces a target without null members + if (to != 2) + { + json patched = source; + patched.merge_patch(patch); + CHECK(patched == target); + } + + // the members of the patch also come in the same order + const auto ordered_source = nested_objects(depth, from); + const auto ordered_target = nested_objects(depth, to); + CHECK(ordered_json::merge_diff(ordered_source, ordered_target).dump() == reference_merge_diff(ordered_source, ordered_target).dump()); + } + } + } + } + + SECTION("a difference only in the innermost value is a patch nested as deeply") + { + for (std::size_t depth = 0; depth <= 300; ++depth) + { + CAPTURE(depth) + json source = {{"v", 1}}; + json target = {{"v", 2}}; + json expected = {{"v", 2}}; + for (std::size_t i = 0; i < depth; ++i) + { + source = json{{"a", std::move(source)}, {"b", i}}; + target = json{{"a", std::move(target)}, {"b", i}}; + expected = json{{"a", std::move(expected)}}; + } + CHECK(json::merge_diff(source, target) == expected); + } + } + + SECTION("values nested too deeply for the call stack") + { + // recursing once per nesting level would exhaust the call stack here. + // The patch is walked rather than compared as a whole, to keep the + // test fast. + const std::size_t depth = 100000; + std::string opening; + for (std::size_t i = 0; i < depth; ++i) + { + opening += R"({"a":)"; + } + const std::string closing(depth, '}'); + const auto source = json::parse(opening + R"({"v":1,"w":0})" + closing); + + const auto patch = json::merge_diff(source, json::parse(opening + R"({"v":2,"w":0})" + closing)); + std::size_t members = 0; + const json* p = &patch; + for (std::size_t i = 0; i < depth; ++i) + { + members += p->size(); + p = &p->at("a"); + } + CHECK(members == depth); + CHECK(*p == json({{"v", 2}})); + + CHECK(json::merge_diff(source, json::parse(opening + R"({"v":1,"w":0})" + closing)).empty()); + } +} diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index ba12eb07b..40b04e383 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -50,11 +50,6 @@ using ordered_json = nlohmann::ordered_json; #include #elif __has_include() #endif - - ///////////////////////////////////////////////////////////////////// - // for #4804 - ///////////////////////////////////////////////////////////////////// - using json_4804 = nlohmann::json::with_binary_t>; #endif #ifdef JSON_HAS_CPP_20 @@ -450,49 +445,6 @@ TEST_CASE("regression tests 3") #endif #ifdef JSON_HAS_CPP_17 - SECTION("issue #4804: from_cbor incompatible with std::vector as binary_t") - { - const std::vector data = {0x80}; - const auto decoded = json_4804::from_cbor(data); - CHECK((decoded == json_4804::array())); - } - -#ifndef SKIP_TESTS_FOR_ENUM_SERIALIZATION - SECTION("discussion #4209 - custom BinaryType direct assignment and round-tripping") - { - // Test that assigning a custom BinaryType directly creates a binary value, not an array - const std::vector original{std::byte{1}, std::byte{2}, std::byte{3}}; - const json_4804 j = original; - CHECK(j.is_binary()); - CHECK(!j.is_array()); - - // Test round-tripping: extracting the binary value back as the custom container type - const auto extracted = j.get>(); - CHECK(extracted == original); - - // Test that the default json alias behavior is unchanged: std::vector -> array - const json default_json = std::vector {1, 2, 3}; - CHECK(default_json.is_array()); - CHECK(!default_json.is_binary()); - } - - SECTION("discussion #4209 - custom BinaryType extraction from parsed array") - { - // Test that extracting a custom BinaryType from a parsed JSON array still works - // (not just from a binary-typed node) - const auto j = json_4804::parse("[1,2,3]"); - CHECK(j.is_array()); - CHECK(!j.is_binary()); - - // Extracting as custom BinaryType should work from arrays - const auto extracted = j.get>(); - CHECK(extracted.size() == 3); - CHECK(extracted[0] == std::byte{1}); - CHECK(extracted[1] == std::byte{2}); - CHECK(extracted[2] == std::byte{3}); - } -#endif - SECTION("issue #5046 - implicit conversion of return json to std::optional no longer implicit") { const json jval{}; diff --git a/tests/src/unit-same-number-types.cpp b/tests/src/unit-same-number-types.cpp new file mode 100644 index 000000000..ba42710c1 --- /dev/null +++ b/tests/src/unit-same-number-types.cpp @@ -0,0 +1,201 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include + +#include +#include +#include +#include +#include + +// basic_json with the same type for number_integer_t and number_unsigned_t +// (#2573): every integer is stored as number_integer + +namespace +{ + +template +using same_types_json = nlohmann::json::with_integers_t; + +using json64 = same_types_json; +using json32 = same_types_json; + +} // namespace + +TEST_CASE_TEMPLATE("same type for number_integer_t and number_unsigned_t", Json, json64, json32) +{ + using integer_t = typename Json::number_integer_t; + using limits = std::numeric_limits; + + SECTION("parsing stores integers as number_integer") + { + const Json j = Json::parse("[0, 1, -1, 2147483647, -2147483648]"); + for (const auto& element : j) + { + CHECK(element.is_number_integer()); + CHECK(!element.is_number_unsigned()); + CHECK(element.type() == nlohmann::detail::value_t::number_integer); + } + CHECK(j[1].template get() == 1); + CHECK(j[2].template get() == -1); + CHECK(j[3].template get() == 2147483647); + CHECK(j[4].template get() == -2147483648LL); + } + + SECTION("integers above the maximum are parsed as floats") + { + // a positive literal is parsed as unsigned first; it must not wrap + // around to a negative number_integer_t + const auto above_max = std::to_string(static_cast(limits::max()) + 1); + const Json j1 = Json::parse(above_max); + CHECK(j1.is_number_float()); + CHECK(j1.template get() == static_cast(static_cast(limits::max()) + 1)); + + const Json j2 = Json::parse("18446744073709551615"); + CHECK(j2.is_number_float()); + CHECK(j2.template get() == 18446744073709551615.0); + + const Json j3 = Json::parse(std::to_string(limits::max())); + CHECK(j3.is_number_integer()); + CHECK(j3.template get() == limits::max()); + } + + SECTION("serialization of the extremes") + { + CHECK(Json(limits::max()).dump() == std::to_string(limits::max())); + // the absolute value of the minimum does not fit into number_unsigned_t + CHECK(Json(limits::min()).dump() == std::to_string(limits::min())); + CHECK(Json::parse(std::to_string(limits::min())) == Json(limits::min())); + CHECK(Json(0).dump() == "0"); + } + + SECTION("construction from unsigned types") + { + const Json j1 = 5u; + CHECK(j1.is_number_integer()); + CHECK(!j1.is_number_unsigned()); + CHECK(j1 == 5); + + const Json j2 = static_cast(200); + CHECK(j2.is_number_integer()); + CHECK(j2.template get() == 200); + + const Json j3 = std::vector {1, 2, 3}; + CHECK(j3.dump() == "[1,2,3]"); + CHECK(j3[0].is_number_integer()); + + const Json j4 = true; + CHECK(j4.is_boolean()); + } + + SECTION("value_t::number_unsigned creates a number_integer") + { + const Json j(nlohmann::detail::value_t::number_unsigned); + CHECK(j.is_number_integer()); + CHECK(!j.is_number_unsigned()); + CHECK(j == 0); + CHECK(j.dump() == "0"); + } + + SECTION("conversions") + { + Json j = 42; + CHECK(j.template get() == 42u); + CHECK(j.template get() == 42u); + CHECK(j.template get() == 42); + CHECK(j.template get() == 42.0); + CHECK(j.template get_ptr() != nullptr); + CHECK(*j.template get_ptr() == 42); + CHECK(j.template get_ref() == 42); + j.template get_ref() = 43; + CHECK(j == 43); + + integer_t i = 0; + j.get_to(i); + CHECK(i == 43); + + const Json object = {{"a", 1}}; + CHECK(object.value("a", 0u) == 1u); + CHECK(object.value("b", 7u) == 7u); + } + + SECTION("conversion from a basic_json with distinct types") + { + const nlohmann::json j = {1, 2u, -3, 4.5}; + CHECK(j[1].is_number_unsigned()); + const Json converted = j; + CHECK(converted.dump() == "[1,2,-3,4.5]"); + CHECK(converted[1].is_number_integer()); + CHECK(!converted[1].is_number_unsigned()); + + const nlohmann::json back = converted; + CHECK(back == j); + } + + SECTION("comparison and hashing") + { + const Json j1 = 1; + const Json j2 = 2u; + CHECK(j1 < j2); + CHECK(j1 != j2); + CHECK(j2 == 2); + CHECK(j2 == 2u); + CHECK(j1 < 1.5); + CHECK(Json(-1) < Json(0u)); + CHECK(std::hash {}(Json(5u)) == std::hash {}(Json(5))); + } + + SECTION("JSON Patch, diff, and flatten") + { + const Json source = Json::parse(R"({"a": [1, 2, 3], "b": 4})"); + const Json target = Json::parse(R"({"a": [1, 5], "c": 6})"); + CHECK(source.patch(Json::diff(source, target)) == target); + CHECK(source.flatten().unflatten() == source); + } + + SECTION("binary formats") + { + const Json j = {{"min", limits::min()}, {"max", limits::max()}, {"zero", 0}, {"small", 100}, {"negative", -100}, {"float", 1.5}}; + + CHECK(Json::from_cbor(Json::to_cbor(j)) == j); + CHECK(Json::from_msgpack(Json::to_msgpack(j)) == j); + CHECK(Json::from_ubjson(Json::to_ubjson(j)) == j); + CHECK(Json::from_bjdata(Json::to_bjdata(j)) == j); + CHECK(Json::from_bson(Json::to_bson(j)) == j); + CHECK(Json::from_bon8(Json::to_bon8(j)) == j); + + for (const auto& element : Json::from_cbor(Json::to_cbor(j))) + { + CHECK(!element.is_number_unsigned()); + } + + // CBOR unsigned integer 2^64-1 does not fit and is read as float + const std::vector cbor_max = {0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + const Json from_cbor = Json::from_cbor(cbor_max); + CHECK(from_cbor.is_number_float()); + CHECK(from_cbor.template get() == 18446744073709551615.0); + + // MessagePack uint 8 is read as number_integer + const std::vector msgpack_uint8 = {0xCC, 0xC8}; + const Json from_msgpack = Json::from_msgpack(msgpack_uint8); + CHECK(from_msgpack.is_number_integer()); + CHECK(from_msgpack == 200); + } +} + +TEST_CASE("same type for number_integer_t and number_unsigned_t: type traits") +{ + CHECK(std::is_same, nlohmann::detail::same_as_number_integer_type>::value); + CHECK(std::is_same, std::uint64_t>::value); + CHECK(nlohmann::detail::is_compatible_number_integer_type::value); + CHECK(!nlohmann::detail::is_compatible_number_integer_type::value); + CHECK(!nlohmann::detail::is_compatible_number_integer_type::value); +} diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index c56c4f5c1..4913c30a6 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -20,8 +20,11 @@ using nlohmann::json; #include +#include #include #include +#include +#include #include "test_utils.hpp" @@ -923,66 +926,68 @@ TEST_CASE("serialization boundary values for the string buffer") // mid-string once fewer than 13 bytes remain (`string_buffer.size() - bytes // < 13`), 13 being one more than the most a single code point can ever // write at once (a surrogate pair: two back-to-back "\uXXXX" escapes, 12 - // bytes). Every write into string_buffer that this check protects happens - // in steps of 2 (a simple "\\x" escape) or 6 (one "\uXXXX" unit), so - // `bytes` only ever takes even values at the point the check runs - the - // tightest values actually reachable are therefore 498 (512 - 498 == 14, - // one simple escape away from the threshold) and 500 (512 - 500 == 12, - // where the flush fires immediately and resets bytes to 0). + // bytes). Characters below 0x60 (control characters, '"' and '\\') are + // escaped straight into the write buffer and never reach string_buffer, so + // under ensure_ascii it fills in steps of 6 (one "\uXXXX" unit for a + // character below U+10000) or 12 (a surrogate pair). With 2-byte characters + // such as U+00E4, `bytes` takes multiples of 6 at the point the check runs; + // the tightest values reachable are therefore 498 (512 - 498 == 14, after + // 83 characters) and 504 (512 - 504 == 8, after 84 characters, where the + // flush fires immediately and resets bytes to 0). - SECTION("a run of 2-byte escapes lands bytes on, and one step past, the flush threshold") + SECTION("a run of 6-byte escapes lands bytes on, and one step past, the flush threshold") { for (const int count : { - 249, 250, 251 + 83, 84, 85 }) { CAPTURE(count) - const json j = std::string(static_cast(count), '\n'); + std::string umlauts; std::string expected = "\""; for (int i = 0; i < count; ++i) { - expected += "\\n"; + umlauts += "\xC3\xA4"; + expected += "\\u00e4"; } expected += '"'; - CHECK(j.dump() == expected); + CHECK(json(umlauts).dump(-1, ' ', true) == expected); } } - SECTION("an ASCII prefix leaves the tightest reachable margin before a 12-byte surrogate pair") + SECTION("a prefix leaves the tightest reachable margin before a 12-byte surrogate pair") { // U+1F600 (the "\xF0\x9F\x98\x80" UTF-8 bytes) is dumped under // ensure_ascii as the 12-byte surrogate pair "\ud83d\ude00"; that // write happens in a single step with no intermediate flush check, so // it is the write most exposed by an off-by-one in the "< 13" guard. - // A prefix of 249 newlines leaves exactly 14 bytes of headroom + // A prefix of 83 U+00E4 characters leaves exactly 14 bytes of headroom // (512 - 498), the smallest margin the guard ever actually allows - // into a new code point; 250 newlines instead trigger the guard's own + // into a new code point; 84 characters instead trigger the guard's own // flush first, so the emoji starts from a freshly emptied (512-byte) - // buffer, and 251 repeats that with one more escape already past the + // buffer, and 85 repeats that with one more escape already past the // reset. Together they cover the margin the guard allows landing on, // one step before, and one step after - all must still produce the // identical, correct escapes. for (const int prefix_count : { - 249, 250, 251 + 83, 84, 85 }) { CAPTURE(prefix_count) - const std::string prefix(static_cast(prefix_count), '\n'); - const std::string emoji = "\xF0\x9F\x98\x80"; - const json j = prefix + emoji; - + std::string prefix; std::string expected_prefix; for (int i = 0; i < prefix_count; ++i) { - expected_prefix += "\\n"; + prefix += "\xC3\xA4"; + expected_prefix += "\\u00e4"; } + const std::string emoji = "\xF0\x9F\x98\x80"; + const json j = prefix + emoji; - // newline escaping does not depend on ensure_ascii: only the - // emoji differs (raw UTF-8 bytes vs. a \u-escaped surrogate pair) + // without ensure_ascii, the whole string is copied unchanged std::string expected_raw = "\""; - expected_raw += expected_prefix; + expected_raw += prefix; expected_raw += emoji; expected_raw += '"'; std::string expected_ascii = "\""; @@ -1027,3 +1032,524 @@ TEST_CASE("serialization boundary values for the string buffer") } } } + +namespace +{ +// appends the escaped form of the ASCII character c to out; written +// independently of the table in serializer::ascii_escapes() +void append_escaped_ascii(std::string& out, const char c, const bool ensure_ascii) +{ + const auto byte = static_cast(c); + switch (byte) + { + case '\b': + out += "\\b"; + break; + case '\t': + out += "\\t"; + break; + case '\n': + out += "\\n"; + break; + case '\f': + out += "\\f"; + break; + case '\r': + out += "\\r"; + break; + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + default: + if (byte < 0x20 || (ensure_ascii && byte == 0x7F)) + { + // byte < 0x80, so the first two hex digits are 0 + const char* const hex = "0123456789abcdef"; + out += "\\u00"; + out += hex[byte >> 4U]; + out += hex[byte & 0x0FU]; + } + else + { + out += c; + } + break; + } +} + +// reference escaper for ASCII strings +std::string reference_escape_ascii(const std::string& s, const bool ensure_ascii) +{ + std::string result = "\""; + for (const char c : s) + { + append_escaped_ascii(result, c, ensure_ascii); + } + result += '"'; + return result; +} +} // namespace + +TEST_CASE("characters below 0x60 are escaped from a table straight into the write buffer") +{ + SECTION("every ASCII character at every position") + { + // lengths up to 17 cover the 8-byte strides of the bulk scanner on + // both sides of the character, so the scanner stops at it from every + // offset + for (std::size_t length = 1; length <= 17; ++length) + { + for (std::size_t pos = 0; pos < length; ++pos) + { + for (unsigned int c = 0; c < 0x80; ++c) + { + std::string s(length, 'a'); + s[pos] = static_cast(c); + CAPTURE(length) + CAPTURE(pos) + CAPTURE(c) + const json j = s; + CHECK(j.dump(-1, ' ', false) == reference_escape_ascii(s, false)); + CHECK(j.dump(-1, ' ', true) == reference_escape_ascii(s, true)); + } + } + } + } + + SECTION("runs of escaped characters") + { + std::string s; + for (unsigned int c = 0; c < 0x20; ++c) + { + s += static_cast(c); + } + s += R"("\"\)"; + CHECK(json(s).dump() == reference_escape_ascii(s, false)); + CHECK(json(s).dump(-1, ' ', true) == reference_escape_ascii(s, true)); + } + + SECTION("escapes straddle the end of the write buffer") + { + // the 8-byte store of a table entry needs room for all 8 bytes, even + // when the escape itself is shorter; a prefix of k plain characters + // shifts the 6-byte escapes across every offset of the 1024-byte + // write buffer + for (std::size_t k = 0; k < 8; ++k) + { + CAPTURE(k) + const std::string s = std::string(k, 'a') + std::string(400, '\x01'); + CHECK(json(s).dump() == reference_escape_ascii(s, false)); + CHECK(json::array({s, s}).dump() == "[" + reference_escape_ascii(s, false) + "," + reference_escape_ascii(s, false) + "]"); + } + } + + SECTION("pending \\u escapes of non-ASCII characters are written first") + { + // under ensure_ascii, U+00E4 is \u-escaped into the string buffer; + // a following escape from the table must not overtake it + const json j = "\xC3\xA4\n\xC3\xA4\"\xF0\x9F\x98\x80\\"; + CHECK(j.dump(-1, ' ', true) == "\"\\u00e4\\n\\u00e4\\\"\\ud83d\\ude00\\\\\""); + CHECK(j.dump(-1, ' ', false) == "\"\xC3\xA4\\n\xC3\xA4\\\"\xF0\x9F\x98\x80\\\\\""); + } + + SECTION("escapes after invalid UTF-8 for every error handler") + { + const json j = "a\xFF\n\xC3\""; + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF", json::type_error&); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"a\\n\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::ignore) == "\"a\\n\\\"\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"a\\ufffd\\n\\ufffd\\\"\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\""); + } +} + +namespace +{ +// expected result of dumping a string: the output, or (with +// error_handler_t::strict) the message of the type_error.316 exception +struct expected_dump +{ + bool throws = false; + std::string text{}; // NOLINT(readability-redundant-member-init) +}; + +std::string upper_hex_byte(const unsigned char byte) +{ + const char* const hex = "0123456789ABCDEF"; + return std::string(1, hex[byte >> 4U]) + hex[byte & 0x0FU]; +} + +void append_u_escape(std::string& out, const std::uint32_t code_unit) +{ + const char* const hex = "0123456789abcdef"; + out += "\\u"; + for (int shift = 12; shift >= 0; shift -= 4) + { + out += hex[(code_unit >> static_cast(shift)) & 0x0FU]; + } +} + +void append_replacement_character(std::string& out, const bool ensure_ascii) +{ + out += ensure_ascii ? "\\ufffd" : "\xEF\xBF\xBD"; +} + +// reference serializer for arbitrary byte strings, written from the documented +// behavior of dump() and independently of serializer::dump_escaped(): UTF-8 is +// decoded strictly (well-formed sequences as in Table 3-7 of the Unicode +// Standard), and each ill-formed sequence is handled as the error handler says +expected_dump reference_dump_string(const std::string& s, const bool ensure_ascii, const json::error_handler_t error_handler) +{ + expected_dump result; + std::string& out = result.text; + out = "\""; + + // handle the ill-formed bytes s[begin, end), of which s[index] is the byte + // that could not be decoded; returns false if an exception is expected + const auto ill_formed = [&](const std::size_t begin, const std::size_t end, const std::size_t index) + { + switch (error_handler) + { + case json::error_handler_t::strict: + result.throws = true; + out = "[json.exception.type_error.316] invalid UTF-8 byte at index " + std::to_string(index) + ": 0x" + upper_hex_byte(static_cast(s[index])); + return false; + case json::error_handler_t::ignore: + break; + case json::error_handler_t::replace: + append_replacement_character(out, ensure_ascii); + break; + case json::error_handler_t::keep: + default: + // kept as is, even with ensure_ascii + out.append(s, begin, end - begin); + break; + } + return true; + }; + + std::size_t i = 0; + while (i < s.size()) + { + const auto lead = static_cast(s[i]); + if (lead < 0x80) + { + append_escaped_ascii(out, s[i], ensure_ascii); + ++i; + continue; + } + + // number of bytes of the sequence and the range of its second byte + std::size_t length = 0; + std::uint32_t code_point = 0; + unsigned char lower = 0x80; + unsigned char upper = 0xBF; + if (lead >= 0xC2 && lead <= 0xDF) + { + length = 2; + code_point = lead & 0x1FU; + } + else if (lead >= 0xE0 && lead <= 0xEF) + { + length = 3; + code_point = lead & 0x0FU; + lower = (lead == 0xE0) ? 0xA0 : 0x80; // no overlong encodings + upper = (lead == 0xED) ? 0x9F : 0xBF; // no surrogates + } + else if (lead >= 0xF0 && lead <= 0xF4) + { + length = 4; + code_point = lead & 0x07U; + lower = (lead == 0xF0) ? 0x90 : 0x80; // no overlong encodings + upper = (lead == 0xF4) ? 0x8F : 0xBF; // nothing above U+10FFFF + } + else + { + // a byte that cannot start a sequence is consumed + if (!ill_formed(i, i + 1, i)) + { + return result; + } + ++i; + continue; + } + + std::size_t j = i + 1; + for (; j < i + length; ++j) + { + if (j == s.size()) + { + // the string ends in the middle of the sequence + switch (error_handler) + { + case json::error_handler_t::strict: + result.throws = true; + out = "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x" + upper_hex_byte(static_cast(s.back())); + return result; + case json::error_handler_t::ignore: + break; + case json::error_handler_t::replace: + append_replacement_character(out, ensure_ascii); + break; + case json::error_handler_t::keep: + default: + out.append(s, i, s.size() - i); + break; + } + out += '"'; + return result; + } + + const auto byte = static_cast(s[j]); + if (byte < lower || byte > upper) + { + break; + } + code_point = (code_point << 6U) | (byte & 0x3FU); + lower = 0x80; + upper = 0xBF; + } + + if (j < i + length) + { + // the sequence was abandoned at s[j]; that byte is read again as + // the start of a new character + if (!ill_formed(i, j, j)) + { + return result; + } + i = j; + continue; + } + + if (!ensure_ascii) + { + out.append(s, i, length); + } + else if (code_point <= 0xFFFF) + { + append_u_escape(out, code_point); + } + else + { + append_u_escape(out, 0xD800U + ((code_point - 0x10000U) >> 10U)); + append_u_escape(out, 0xDC00U + ((code_point - 0x10000U) & 0x3FFU)); + } + i = j; + } + + out += '"'; + return result; +} + +std::string hex_dump(const std::string& s) +{ + std::string result; + for (const char c : s) + { + result += upper_hex_byte(static_cast(c)); + result += ' '; + } + return result; +} + +// compares dump() of s with the reference for every error handler, with +// ensure_ascii on and off; CHECKs only on a mismatch to keep the sweep fast; +// error_handler_t::strict can be skipped where the position of the string +// buffer does not matter, as throwing is slow on some platforms +void check_dump_against_reference(const std::string& s, const bool with_strict = true) +{ + const json j = s; + const std::array error_handlers = + { + { + json::error_handler_t::strict, json::error_handler_t::ignore, + json::error_handler_t::replace, json::error_handler_t::keep + } + }; + for (const auto error_handler : error_handlers) + { + if (!with_strict && error_handler == json::error_handler_t::strict) + { + continue; + } + for (const bool ensure_ascii : + { + false, true + }) + { + const expected_dump expected = reference_dump_string(s, ensure_ascii, error_handler); + expected_dump actual; + int id = 0; + try + { + actual.text = j.dump(-1, ' ', ensure_ascii, error_handler); + } + catch (const json::exception& e) + { + actual.throws = true; + actual.text = e.what(); + id = e.id; + } + if (actual.throws != expected.throws || actual.text != expected.text || (actual.throws && id != 316)) + { + const std::string input = hex_dump(s); + CAPTURE(input) + CAPTURE(ensure_ascii) + CAPTURE(static_cast(error_handler)) + CHECK(actual.throws == expected.throws); + CHECK(actual.text == expected.text); + CHECK((!actual.throws || id == 316)); + } + } + } +} +} // namespace + +TEST_CASE("invalid UTF-8 at every position is handled like the reference") +{ + // ill-formed byte sequences; those that end in a truncated sequence are + // also cut short by the filler that follows, or by the end of the string + const std::vector patterns = + { + // stray continuation bytes + "\x80", "\xBF", + // bytes that cannot start a sequence + "\xC0", "\xC1", "\xF5", "\xF6", "\xF7", "\xF8", "\xF9", "\xFA", "\xFB", "\xFC", "\xFD", "\xFE", "\xFF", + // overlong encodings + "\xC0\xAF", "\xE0\x80\x80", "\xF0\x80\x80\x80", + // UTF-16 surrogates + "\xED\xA0\x80", "\xED\xBF\xBF", + // a code point above U+10FFFF + "\xF4\x90\x80\x80", + // sequences cut short by ASCII, a control character, a quote, a + // backslash, DEL, or the lead byte of a well-formed character + "\xC3" "a", "\xE2\x82" "\n", "\xE2\x82\"", "\xF0\x9F\x98\\", "\xC3\x7F", "\xC3\xC3\xA4", "\xE2\xF0\x9F\x98\x80", + // truncated sequences + "\xC3", "\xE2\x82", "\xF0\x9F\x98" + }; + + // well-formed characters of 1, 2, 3, and 4 bytes + const std::vector fillers = {"a", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x98\x80"}; + + const auto make_string = [](const std::string & filler, const std::size_t before, const std::string & pattern, const std::size_t after) + { + std::string s; + for (std::size_t k = 0; k < before; ++k) + { + s += filler; + } + s += pattern; + for (std::size_t k = 0; k < after; ++k) + { + s += filler; + } + return s; + }; + + SECTION("the reference") + { + // the reference itself, against hand-written results + const std::string s = "a\xFF\n\xC3\"\xE0\x80\xF0\x9F\x98\x80\xE2\x82"; + CHECK(reference_dump_string(s, false, json::error_handler_t::strict).text == "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF"); + CHECK(reference_dump_string(s.substr(2), false, json::error_handler_t::strict).text == "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0x22"); + CHECK(reference_dump_string("\xF0\x9F\x98\x80\xE2\x82", true, json::error_handler_t::strict).text == "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x82"); + CHECK(reference_dump_string(s, false, json::error_handler_t::ignore).text == "\"a\\n\\\"\xF0\x9F\x98\x80\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::ignore).text == "\"a\\n\\\"\\ud83d\\ude00\""); + CHECK(reference_dump_string(s, false, json::error_handler_t::replace).text == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\xEF\xBF\xBD\xEF\xBF\xBD\xF0\x9F\x98\x80\xEF\xBF\xBD\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::replace).text == "\"a\\ufffd\\n\\ufffd\\\"\\ufffd\\ufffd\\ud83d\\ude00\\ufffd\""); + CHECK(reference_dump_string(s, false, json::error_handler_t::keep).text == "\"" + std::string("a\xFF\\n\xC3\\\"\xE0\x80\xF0\x9F\x98\x80\xE2\x82") + "\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::keep).text == "\"" + std::string("a\xFF\\n\xC3\\\"\xE0\x80\\ud83d\\ude00\xE2\x82") + "\""); + check_dump_against_reference(s); + } + + SECTION("short strings") + { + // strings of up to 17 bytes put the pattern at every offset of an + // 8-byte stride of the bulk scanners, with whole strides on both + // sides; lengths around 24, 32, and 40 bytes cross the 16- and 32-byte + // strides; error_handler_t::strict is checked for up to 17 bytes of + // ASCII and 8 bytes of other characters only, as throwing is slow on + // some platforms + for (const auto& filler : fillers) + { + for (std::size_t length = 0; length * filler.size() <= 40; ++length) + { + const std::size_t bytes = length * filler.size(); + if (filler.size() == 1 && bytes > 17 && (bytes + 1) % 8 > 2) + { + continue; + } + for (std::size_t pos = 0; pos <= length; ++pos) + { + for (const auto& pattern : patterns) + { + check_dump_against_reference(make_string(filler, pos, pattern, length - pos), bytes <= (filler.size() == 1 ? 17 : 8)); + } + } + } + } + } + + SECTION("strings crossing the end of the string buffer") + { + // characters that are not copied in bulk go through a 512-byte string + // buffer that is written out once fewer than 13 bytes are left; with + // ensure_ascii, each 2-byte character takes 6 bytes of it and each + // 4-byte character 12, so the pattern meets the end of the buffer after + // about 84 or 42 characters (3-byte characters behave like 2-byte + // ones); plain ASCII, copied in bulk, is checked around 512 bytes; + // positions further away are sampled; error_handler_t::strict throws + // at the first ill-formed byte, regardless of the string buffer, and is + // covered above + for (const auto& filler : fillers) + { + if (filler.size() == 3) + { + continue; + } + const std::size_t escaped_size = filler.size() == 1 ? 1 : filler.size() * 3; + const std::size_t middle = 506 / escaped_size; + const std::size_t window = filler.size() == 1 ? 8 : 16; + for (std::size_t pos = 0; pos <= middle + window; pos += (pos + window >= middle) ? 1 : 29) + { + for (const std::size_t after : + { + std::size_t{0}, std::size_t{12} + }) + { + for (const auto& pattern : patterns) + { + check_dump_against_reference(make_string(filler, pos, pattern, after), false); + } + } + } + } + } + + SECTION("repeated invalid sequences at every offset of the string buffer") + { + // ill-formed sequences that are kept or replaced fill the string + // buffer without a bulk copy in between; a prefix of 0 to 12 bytes + // 0xFF (kept as 1 byte, replaced by 3 or 6) shifts them across every + // offset of the end of the buffer + for (const auto& pattern : patterns) + { + for (std::size_t shift = 0; shift <= 12; ++shift) + { + std::string s(shift, '\xFF'); + while (s.size() < 600) + { + s += pattern; + } + check_dump_against_reference(s, shift == 0); + check_dump_against_reference(s + "\xF0\x9F\x98\x80", false); + } + } + } +} diff --git a/tests/src/unit-throw_messages.cpp b/tests/src/unit-throw_messages.cpp new file mode 100644 index 000000000..1c2289626 --- /dev/null +++ b/tests/src/unit-throw_messages.cpp @@ -0,0 +1,135 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include + +// The exceptions in hot core functions are constructed out of line. These tests pin the +// exact what() texts of the sites that moved, so the refactoring cannot change them. + +#define CHECK_MSG(expr, msg) CHECK_THROWS_WITH(expr, msg) + +TEST_CASE("exception messages of out-of-line throw sites") +{ + json arr = {1, 2, 3}; + json arr2 = {4, 5}; + json obj = {{"a", 1}, {"b", 2}}; + json obj2 = {{"c", 3}}; + json num = 1; + + SECTION("iterators") + { + CHECK_MSG((void)(arr.begin() == arr2.begin()), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers"); + CHECK_MSG((void)(arr.begin() < arr2.begin()), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers"); + CHECK_MSG((void)(obj.begin() < obj.end()), "[json.exception.invalid_iterator.213] cannot compare order of object iterators"); + CHECK_MSG(obj.begin() += 1, "[json.exception.invalid_iterator.209] cannot use offsets with object iterators"); + CHECK_MSG((void)(obj.begin() - 1), "[json.exception.invalid_iterator.209] cannot use offsets with object iterators"); + CHECK_MSG((void)obj.begin()[0], "[json.exception.invalid_iterator.208] cannot use operator[] for object iterators"); + CHECK_MSG((void)arr.begin().key(), "[json.exception.invalid_iterator.207] cannot use key() for non-object iterators"); + json null; + CHECK_MSG((void)*null.end(), "[json.exception.invalid_iterator.214] cannot get value"); + } + + SECTION("constructors") + { + CHECK_MSG(json(arr.begin(), arr2.end()), "[json.exception.invalid_iterator.201] iterators are not compatible"); + CHECK_MSG(json(num.begin() + 1, num.end() + 1), "[json.exception.invalid_iterator.204] iterators out of range"); + CHECK_MSG(json(json::initializer_list_t{1, 2}, false, json::value_t::object), "[json.exception.type_error.301] cannot create object from initializer list"); + } + + SECTION("get_ref and operator[]") + { + CHECK_MSG((void)num.get_ref(), "[json.exception.type_error.303] incompatible ReferenceType for get_ref, actual type is number"); + CHECK_MSG((void)num[0], "[json.exception.type_error.305] cannot use operator[] with a numeric argument with number"); + } + + SECTION("at") + { + const json carr = arr; + const json cobj = obj; + CHECK_MSG((void)arr.at(static_cast(10)), "[json.exception.out_of_range.401] array index 10 is out of range"); + CHECK_MSG((void)carr.at(static_cast(3)), "[json.exception.out_of_range.401] array index 3 is out of range"); + CHECK_MSG((void)num.at(static_cast(0)), "[json.exception.type_error.304] cannot use at() with number"); + CHECK_MSG((void)obj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); + CHECK_MSG((void)cobj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); + CHECK_MSG((void)obj.at(std::string("zz")), "[json.exception.out_of_range.403] key 'zz' not found"); + CHECK_MSG((void)num.at("a"), "[json.exception.type_error.304] cannot use at() with number"); + + nlohmann::ordered_json oobj = {{"a", 1}}; + const nlohmann::ordered_json coobj = oobj; + CHECK_MSG((void)oobj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); + CHECK_MSG((void)coobj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); + CHECK_MSG((void)coobj.at(std::string("zz")), "[json.exception.out_of_range.403] key 'zz' not found"); + } + + SECTION("erase") + { + CHECK_MSG(arr.erase(arr2.begin()), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(num.erase(num.end()), "[json.exception.invalid_iterator.205] iterator out of range"); + CHECK_MSG(arr.erase(arr.begin(), arr2.end()), "[json.exception.invalid_iterator.203] iterators do not fit current value"); + CHECK_MSG(num.erase(num.begin() + 1, num.end()), "[json.exception.invalid_iterator.204] iterators out of range"); + CHECK_MSG(arr.erase(static_cast(10)), "[json.exception.out_of_range.401] array index 10 is out of range"); + } + + SECTION("insert") + { + CHECK_MSG(arr.insert(arr2.begin(), 1), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(arr.insert(arr2.begin(), 2, 1), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(arr.insert(arr2.begin(), arr2.begin(), arr2.end()), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(arr.insert(arr.begin(), arr.begin(), arr2.end()), "[json.exception.invalid_iterator.210] iterators do not fit"); + CHECK_MSG(arr.insert(arr.begin(), arr.begin(), arr.end()), "[json.exception.invalid_iterator.211] passed iterators may not belong to container"); + CHECK_MSG(arr.insert(arr.begin(), obj.begin(), obj.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to arrays"); + CHECK_MSG(arr.insert(arr2.begin(), {1, 2}), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(obj.insert(obj2.begin(), arr.end()), "[json.exception.invalid_iterator.210] iterators do not fit"); + CHECK_MSG(obj.insert(arr.begin(), arr.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to objects"); + CHECK_MSG(obj.update(arr.begin(), arr2.end()), "[json.exception.invalid_iterator.210] iterators do not fit"); + } + + SECTION("patch") + { + const auto patch_msg = [](const char* doc, const char* patch) + { + try + { + json d = json::parse(doc); + d.patch(json::parse(patch)); + } + catch (const json::exception& e) + { + // with JSON_DIAGNOSTICS, what() names the offending value like "(/0) "; drop it + std::string msg = e.what(); + const auto pos = msg.find(" (/"); + if (pos != std::string::npos) + { + msg.erase(pos + 1, msg.find(") ", pos) - pos + 1); + } + return msg; + } + return std::string(); + }; + + CHECK(patch_msg("{}", "{}") == "[json.exception.parse_error.104] parse error: JSON patch must be an array of objects"); + CHECK(patch_msg("{}", "[1]") == "[json.exception.parse_error.104] parse error: JSON patch must be an array of objects"); + CHECK(patch_msg("{}", "[{}]") == "[json.exception.parse_error.105] parse error: operation must have member 'op'"); + CHECK(patch_msg("{}", "[{\"op\":1}]") == "[json.exception.parse_error.105] parse error: operation must have string member 'op'"); + CHECK(patch_msg("{}", "[{\"op\":\"add\"}]") == "[json.exception.parse_error.105] parse error: operation 'add' must have member 'path'"); + CHECK(patch_msg("{}", "[{\"op\":\"add\",\"path\":1}]") == "[json.exception.parse_error.105] parse error: operation 'add' must have string member 'path'"); + CHECK(patch_msg("{}", "[{\"op\":\"foo\",\"path\":\"\"}]") == "[json.exception.parse_error.105] parse error: operation value 'foo' is invalid"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"remove\",\"path\":\"/b\"}]") == "[json.exception.out_of_range.403] key 'b' not found"); + CHECK(patch_msg("[1]", "[{\"op\":\"add\",\"path\":\"/5\",\"value\":0}]") == "[json.exception.out_of_range.401] array index 5 is out of range"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"add\",\"path\":\"/a/b\",\"value\":0}]") == "[json.exception.out_of_range.411] cannot add value: the JSON Patch 'add' target's parent is of type number, but must be an object or array"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"remove\",\"path\":\"/a/b\"}]") == "[json.exception.out_of_range.413] cannot remove value: the JSON Patch 'remove' target's parent is of type number, but must be an object or array"); + CHECK(patch_msg("{\"a\":{}}", "[{\"op\":\"move\",\"from\":\"/a\",\"path\":\"/a/b\"}]") == "[json.exception.out_of_range.414] cannot move value: 'from' path '/a' is a proper prefix of 'path' '/a/b'"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"test\",\"path\":\"/a\",\"value\":2}]") == "[json.exception.other_error.501] unsuccessful: {\"op\":\"test\",\"path\":\"/a\",\"value\":2}"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"move\",\"path\":\"/a\"}]") == "[json.exception.parse_error.105] parse error: operation 'move' must have member 'from'"); + } +} diff --git a/tests/src/unit-to_chars.cpp b/tests/src/unit-to_chars.cpp index def2793c2..95c5f2bd0 100644 --- a/tests/src/unit-to_chars.cpp +++ b/tests/src/unit-to_chars.cpp @@ -15,6 +15,21 @@ #include using nlohmann::detail::dtoa_impl::reinterpret_bits; +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(JSON_HAS_CPP_17) + #include +#endif + namespace { float make_float(uint32_t sign_bit, uint32_t biased_exponent, uint32_t significand) @@ -450,7 +465,7 @@ TEST_CASE("formatting") check_double( 1.2345e+18, "1.2345e+18" ); // 1.2345e+18 1.2345e+18 1.2345e18 check_double( 1.2345e+19, "1.2345e+19" ); // 1.2345e+19 1.2345e+19 1.2345e19 check_double( 1.2345e+20, "1.2345e+20" ); // 1.2345e+20 1.2345e+20 1.2345e20 - check_double( 1.2345e+21, "1.2344999999999999e+21" ); // 1.2345e+21 1.2344999999999999e+21 1.2345e21 + check_double( 1.2345e+21, "1.2345e+21" ); // 1.2345e+21 1.2344999999999999e+21 1.2345e21 check_double( 1.2345e+22, "1.2345e+22" ); // 1.2345e+22 1.2345e+22 1.2345e22 } @@ -514,3 +529,320 @@ TEST_CASE("formatting") check_integer(1000000000000000000LL, "1000000000000000000"); } } + +namespace +{ +// a small unsigned big integer (32-bit limbs, least significant first), to +// recompute the powers of ten of the shortest double conversion +using big = std::vector; + +void big_mul_small(big& x, std::uint32_t m) +{ + std::uint64_t carry = 0; + for (auto& limb : x) + { + const std::uint64_t v = (static_cast(limb) * m) + carry; + limb = static_cast(v); + carry = v >> 32u; + } + if (carry != 0) + { + x.push_back(static_cast(carry)); + } +} + +void big_div_small(big& x, std::uint32_t d) +{ + std::uint64_t rest = 0; + for (std::size_t i = x.size(); i-- > 0;) + { + const std::uint64_t v = (rest << 32u) | x[i]; + x[i] = static_cast(v / d); + rest = v % d; + } + while (!x.empty() && x.back() == 0) + { + x.pop_back(); + } +} + +std::size_t big_bit_length(const big& x) +{ + std::size_t n = 32 * x.size(); + for (std::uint32_t top = x.back(); (top & 0x80000000u) == 0; top <<= 1u) + { + --n; + } + return n; +} + +bool big_bit(const big& x, std::size_t i) +{ + return ((x[i / 32] >> (i % 32)) & 1u) != 0; +} + +/// the 128 most significant bits of x (floor), shifted left if x has fewer bits +std::pair big_top128(const big& x) +{ + const std::size_t n = big_bit_length(x); + std::uint64_t high = 0; + std::uint64_t low = 0; + for (std::size_t k = 0; k < 128; ++k) + { + const bool bit = k < n && big_bit(x, n - 1 - k); + if (k < 64) + { + high = (high << 1u) | (bit ? 1u : 0u); + } + else + { + low = (low << 1u) | (bit ? 1u : 0u); + } + } + return {high, low}; +} + +/// the digits (without trailing zeros) and the decimal exponent of a +/// representation "[-]d[.ddd][e[+-]x]" +std::pair digits_and_exponent(const std::string& s) +{ + std::string digits; + int point = -1; + int exponent = 0; + for (std::size_t i = 0; i < s.size(); ++i) + { + const char c = s[i]; + if (c >= '0' && c <= '9') + { + digits += c; + } + else if (c == '.') + { + point = static_cast(digits.size()); + } + else if (c == 'e' || c == 'E') + { + exponent = std::stoi(s.substr(i + 1)); + break; + } + } + int e = exponent + (point < 0 ? static_cast(digits.size()) : point) - static_cast(digits.size()); + const std::size_t first = digits.find_first_not_of('0'); + digits = first == std::string::npos ? "0" : digits.substr(first); + while (digits.size() > 1 && digits.back() == '0') + { + digits.pop_back(); + ++e; + } + return {digits, e}; +} + +/// the correctly rounded double of a decimal text +/// (not std::strtod: the C runtimes of some platforms, e.g. MinGW's, round +/// some 16 and 17 digit inputs wrongly) +double parse_double(const std::string& text) +{ + const nlohmann::json j = nlohmann::json::parse(text, nullptr, false); + // (a discarded value: out of range, as strtod's HUGE_VAL) + return j.is_discarded() ? std::numeric_limits::infinity() : j.get(); +} + +/// whether the decimal digits * 10^e reads back as v +bool reads_back(const std::string& digits, int e, double v) +{ + const std::string text = digits + "e" + std::to_string(e); + // (compared bit for bit: v is positive and finite, and -Wfloat-equal) + return reinterpret_bits(parse_double(text)) == reinterpret_bits(v); +} + +/// Check the representation of a positive finite double: it reads back as +/// the same value, and no representation with fewer digits does. +void check_shortest(double v) +{ + std::array buf{}; + char* end = nlohmann::detail::to_chars(buf.data(), buf.data() + 32, v); + const std::string text(buf.data(), end); + CAPTURE(text) + CHECK(parse_double(text) == v); + // the layout is that of format_buffer() for the digits of Zmij + const auto sd = nlohmann::detail::zmij::to_shortest(reinterpret_bits(v)); + const std::uint64_t significand = sd.has_digit ? (sd.integral * 10) + sd.digit : sd.integral; + int exponent = sd.has_digit ? sd.exponent : sd.exponent + 1; + std::string significand_digits = std::to_string(significand); + while (significand_digits.size() > 1 && significand_digits.back() == '0') + { + significand_digits.pop_back(); + ++exponent; + } + std::array reference{}; + std::copy(significand_digits.begin(), significand_digits.end(), reference.begin()); + const char* const reference_end = nlohmann::detail::dtoa_impl::format_buffer(reference.data(), static_cast(significand_digits.size()), exponent, -4, 15); + CHECK(text == std::string(reference.data(), static_cast(reference_end - reference.data()))); + // and write_positive() is what to_chars() calls + std::array positive{}; + const char* const positive_end = nlohmann::detail::dtoa_impl::write_positive(positive.data(), positive.data() + positive.size(), v); + CHECK(text == std::string(positive.data(), static_cast(positive_end - positive.data()))); + const auto de = digits_and_exponent(text); + const std::string& digits = de.first; + if (digits.size() > 1) + { + // the decimals of one digit fewer next to the value: v lies within + // half a unit of the last digit of text, so the closest ones are the + // truncated digits and their successor + // (not a stream with std::setprecision(digits.size() - 2): the + // standard library of MSVC 2017 ignores a precision of 0) + const std::string m = digits.substr(0, digits.size() - 1); + const int e = de.second + 1; + const std::uint64_t mid = std::stoull(m); + for (const std::uint64_t candidate : + { + mid - 1, mid, mid + 1 + }) + { + CAPTURE(candidate) + CHECK(!reads_back(std::to_string(candidate), e, v)); + } + } + // (icpc with libstdc++ 11 defines __cpp_lib_to_chars, but has no floating-point std::to_chars) +#if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars) && !defined(__INTEL_COMPILER) + // the closest of the shortest representations, as std::to_chars finds it + std::array std_text{}; + const auto r = std::to_chars(std_text.data(), std_text.data() + std_text.size(), v, std::chars_format::scientific); + CHECK(digits_and_exponent(std::string(std_text.data(), r.ptr)) == de); +#endif +} +} // namespace + +TEST_CASE("shortest digits of doubles") +{ + SECTION("powers of ten") + { + // the 128-bit significands of 10^k, rounded down, recomputed + for (int k = -342; k <= 341; ++k) + { + CAPTURE(k) + big x{1}; + if (k >= 0) + { + for (int i = 0; i < k; ++i) + { + big_mul_small(x, 10); + } + } + else + { + // floor(2^b / 10^-k) for a b that leaves more than 128 bits + const int b = 128 + 64 + (4 * -k); + x.assign(static_cast(b / 32) + 1, 0); + x.back() = 1u << (b % 32); + for (int i = 0; i < -k; ++i) + { + big_div_small(x, 10); + } + } + const auto expected = big_top128(x); + const auto actual = nlohmann::detail::zmij::pow10(k); + CHECK(actual.high == expected.first); + CHECK(actual.low == expected.second); + } + } + + SECTION("boundary values") + { + for (const double v : + { + std::numeric_limits::min(), std::numeric_limits::max(), std::numeric_limits::denorm_min(), + std::nextafter(std::numeric_limits::min(), 0.0), 1.0, 2.0, 0.1, 0.3, 1e21, 1e22, 1e23, 5e-324, 9007199254740993.0, + 1.2345e+21, 2.2250738585072014e-308, 1.7976931348623157e308, 4.9406564584124654e-324, 123456789012345680.0 + }) + { + check_shortest(v); + } + // all powers of two (their rounding interval is narrower below) + for (int e = -1074; e <= 1023; ++e) + { + check_shortest(std::ldexp(1.0, e)); + } + // powers of ten and their neighbors + for (int e = -323; e <= 308; ++e) + { + const double p = std::strtod(("1e" + std::to_string(e)).c_str(), nullptr); + check_shortest(p); + check_shortest(std::nextafter(p, 0.0)); + check_shortest(std::nextafter(p, std::numeric_limits::infinity())); + } + } + + SECTION("random doubles") + { + std::mt19937_64 rng(5295); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (int i = 0; i < 100000; ++i) + { + const std::uint64_t bits = rng() & 0x7FFFFFFFFFFFFFFFu; + const auto v = reinterpret_bits(bits); + if (std::isfinite(v) && bits != 0) + { + check_shortest(v); + } + } + } +} + +TEST_CASE("choice of the conversion") +{ + using nlohmann::detail::dtoa_impl::is_binary64; + + SECTION("by the format of the type") + { + // Zmij needs binary64 numbers; everything else uses Grisu2 + static_assert(!is_binary64::value, "float is not binary64"); + static_assert(is_binary64::value == (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53), + "double is binary64 where it is IEEE 754 with 53 digits"); + static_assert(!is_binary64::value, "integers are not binary64"); + static_assert(is_binary64::value == (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && sizeof(long double) == 8), + "long double is binary64 where it has the format of a double"); + CHECK(!is_binary64::value); + CHECK(is_binary64::value); + } + + SECTION("float: Grisu2, double: Zmij") + { + // 5.3165205877497296e+16 is one of the doubles for which Grisu2 does not find the shortest digits + constexpr double value = 5.3165205877497296e+16; + std::array buf{}; + const char* const last = buf.data() + buf.size(); + + char* end = nlohmann::detail::dtoa_impl::write_positive(buf.data(), last, value); + CHECK(std::string(buf.data(), end) == "5.31652058774973e+16"); + end = nlohmann::detail::dtoa_impl::write_positive_grisu2(buf.data(), last, value); + CHECK(std::string(buf.data(), end) == "5.3165205877497296e+16"); + + constexpr float f = 1.1754944e-38f; + end = nlohmann::detail::dtoa_impl::write_positive(buf.data(), last, f); + const std::string dispatched(buf.data(), end); + end = nlohmann::detail::dtoa_impl::write_positive_grisu2(buf.data(), last, f); + CHECK(dispatched == std::string(buf.data(), end)); + } + + SECTION("long double with the format of a double: Zmij") + { + // (on platforms where long double is wider, Grisu2 does not apply either: the snprintf fallback does) + // (constant on a given platform) + DOCTEST_MSVC_SUPPRESS_WARNING_PUSH + DOCTEST_MSVC_SUPPRESS_WARNING(4127) + if (std::numeric_limits::digits == 53 && std::numeric_limits::is_iec559) + { + using long_double_json = nlohmann::json::with_float_t; + for (const double d : + { + 5.3165205877497296e+16, 1.0, 0.1, 123456.789, 2.2250738585072014e-308, 1.7976931348623157e+308, -5.3165205877497296e+16 + }) + { + CAPTURE(d) + CHECK(long_double_json(static_cast(d)).dump() == nlohmann::json(d).dump()); + } + CHECK(long_double_json(5.3165205877497296e+16L).dump() == "5.31652058774973e+16"); + } + DOCTEST_MSVC_SUPPRESS_WARNING_POP + } +} diff --git a/tests/src/unit-to_json-rvalue.cpp b/tests/src/unit-to_json-rvalue.cpp new file mode 100644 index 000000000..261493f7a --- /dev/null +++ b/tests/src/unit-to_json-rvalue.cpp @@ -0,0 +1,257 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) + #if __has_include() + #include + #define TO_JSON_RVALUE_HAS_FS 1 + #endif +#endif + +namespace +{ +// element type that counts how it is converted to JSON: a conversion from an +// lvalue copies, a conversion from an rvalue moves +struct tracked +{ + std::string payload; + bool moved_from = false; + + explicit tracked(std::string p) : payload(std::move(p)) {} + tracked(const tracked&) = default; + tracked& operator=(const tracked&) = default; + tracked(tracked&& other) noexcept : payload(std::move(other.payload)), moved_from(other.moved_from) + { + other.moved_from = true; + } + tracked& operator=(tracked&& other) noexcept + { + payload = std::move(other.payload); + moved_from = other.moved_from; + other.moved_from = true; + return *this; + } + ~tracked() = default; +}; + +int tracked_copies = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables,misc-use-internal-linkage) +int tracked_moves = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables,misc-use-internal-linkage) + +void to_json(json& j, const tracked& t) +{ + ++tracked_copies; + j = t.payload; +} + +void to_json(json& j, tracked&& t) +{ + ++tracked_moves; + j = std::move(t.payload); + t.moved_from = true; +} + +// user type with its own rvalue-aware serializer +struct udt_container +{ + std::vector values; +}; +} // namespace + +namespace nlohmann +{ +template<> +struct adl_serializer +{ + static void to_json(json& j, const udt_container& c) + { + j = c.values; + j.push_back("copied"); + } + static void to_json(json& j, udt_container&& c) + { + j = std::move(c.values); + j.push_back("moved"); + } +}; +} // namespace nlohmann + +TEST_CASE("to_json from rvalue containers") +{ + const std::string long_a(64, 'a'); + const std::string long_b(64, 'b'); + + SECTION("vector: elements are moved") + { + std::vector v{long_a, long_b}; + const json j = std::move(v); + CHECK(j == json::array({long_a, long_b})); + for (const auto& s : v) // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved) + { + CHECK(s.empty()); // long strings leave the source element empty when moved + } + } + + SECTION("vector: lvalues are copied and left untouched") + { + const std::vector v{long_a, long_b}; + const json j = v; + CHECK(j == json::array({long_a, long_b})); + CHECK(v[0] == long_a); + CHECK(v[1] == long_b); + + std::vector v2{long_a, long_b}; + const json j2 = v2; + CHECK(j2 == j); + CHECK(v2[0] == long_a); + CHECK(v2[1] == long_b); + } + + SECTION("tracked elements: rvalue moves, lvalue copies") + { + tracked_copies = 0; + tracked_moves = 0; + + std::vector v; + v.emplace_back("x"); + v.emplace_back("y"); + + const json copy = v; + CHECK(copy == json::array({"x", "y"})); + CHECK(tracked_copies == 2); + CHECK(tracked_moves == 0); + CHECK_FALSE(v[0].moved_from); + + const json moved = std::move(v); + CHECK(moved == copy); + CHECK(tracked_copies == 2); + CHECK(tracked_moves == 2); + } + + SECTION("const rvalue is copied") + { + tracked_copies = 0; + tracked_moves = 0; + const std::vector v(2, tracked("z")); + const json j = std::move(v); // NOLINT(performance-move-const-arg,hicpp-move-const-arg) + CHECK(j == json::array({"z", "z"})); + CHECK(tracked_copies == 2); + CHECK(tracked_moves == 0); + } + + SECTION("other sequence containers") + { + CHECK(json(std::deque {long_a, long_b}) == json::array({long_a, long_b})); + CHECK(json(std::list {long_a, long_b}) == json::array({long_a, long_b})); + CHECK(json(std::forward_list {long_a, long_b}) == json::array({long_a, long_b})); + CHECK(json(std::array {{long_a, long_b}}) == json::array({long_a, long_b})); + CHECK(json(std::set {long_a, long_b}) == json::array({long_a, long_b})); + CHECK(json(std::vector {true, false}) == json::array({true, false})); + CHECK(json(std::vector {1, 2, 3}) == json::array({1, 2, 3})); + CHECK(json(std::vector> {{long_a}, {long_b}}) == json::array({json::array({long_a}), json::array({long_b})})); + CHECK(json(std::vector {1, "a"}) == json::array({1, "a"})); + } + + SECTION("valarray") + { + std::valarray va(3); + va[0] = 1; + va[1] = 2; + va[2] = 3; + CHECK(json(std::move(va)) == json::array({1, 2, 3})); // NOLINT(performance-move-const-arg,hicpp-move-const-arg) + CHECK(json(std::valarray(2)) == json::array({0, 0})); + } + + SECTION("array_t and json arrays") + { + json::array_t arr{1, 2}; + CHECK(json(std::move(arr)) == json::array({1, 2})); + json a = json::array({1, 2}); + const json b = std::move(a); + CHECK(b == json::array({1, 2})); + } + + SECTION("object-like containers") + { + std::map m{{"a", long_a}, {"b", long_b}}; + const json j = std::move(m); + CHECK(j == json({{"a", long_a}, {"b", long_b}})); + for (const auto& kv : m) // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved) + { + CHECK(kv.second.empty()); // values moved, keys (const) copied + } + + const std::map cm{{"a", long_a}}; + const json jc = cm; + CHECK(jc["a"] == long_a); + CHECK(cm.at("a") == long_a); + + std::unordered_map um{{"a", long_a}}; + CHECK(json(std::move(um)) == json({{"a", long_a}})); + + std::multimap mm{{"a", 1}}; + CHECK(json(std::move(mm)) == json({{"a", 1}})); + + tracked_copies = 0; + tracked_moves = 0; + std::map mt; + mt.emplace("k", tracked("v")); + const json jt = std::move(mt); + CHECK(jt == json({{"k", "v"}})); + CHECK(tracked_moves == 1); + CHECK(tracked_copies == 0); + + json::object_t obj{{"a", 1}}; + CHECK(json(std::move(obj)) == json({{"a", 1}})); + } + + SECTION("user type with its own to_json overloads") + { + udt_container c{{1, 2}}; + const json lv = c; + CHECK(lv == json::array({1, 2, "copied"})); + const json rv = std::move(c); + CHECK(rv == json::array({1, 2, "moved"})); + } + + SECTION("assignment and implicit conversion") + { + std::vector v{long_a}; + json j; + j = std::move(v); + CHECK(j == json::array({long_a})); + json k = {std::vector{long_b}}; + CHECK(k.size() == 1); + } + +#ifdef TO_JSON_RVALUE_HAS_FS + SECTION("std::filesystem::path rvalue is still a string") + { + CHECK(json(std::filesystem::path("a/b")) == json("a/b")); +#ifdef __cpp_lib_char8_t + CHECK(json(std::u8string(u8"a/b")) == json("a/b")); +#endif + } +#endif +}