diff --git a/.github/workflows/cancel_closed_pr_runs.yml b/.github/workflows/cancel_closed_pr_runs.yml index 3d3a3dd57..99010d81b 100644 --- a/.github/workflows/cancel_closed_pr_runs.yml +++ b/.github/workflows/cancel_closed_pr_runs.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index f4a546a2b..ba60d70e9 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -31,7 +31,7 @@ jobs: mkdir -p ./pr echo ${{ github.event.number }} > ./pr/number echo ${{ github.event.pull_request.user.login }} > ./pr/author - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: pr path: pr/ @@ -46,7 +46,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -170,7 +170,7 @@ jobs: # instead of installing the pinned astyle version locally. - name: Upload patch if: steps.diff.outputs.has_diff == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: amalgamation-patch path: patch/amalgamation.patch diff --git a/.github/workflows/check_docs_links.yml b/.github/workflows/check_docs_links.yml index 425a9b03e..bf174c245 100644 --- a/.github/workflows/check_docs_links.yml +++ b/.github/workflows/check_docs_links.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 30 steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index e523eddbc..43efd0e16 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -35,7 +35,7 @@ jobs: dry-run: false language: c++ - name: Upload Crash - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: failure() && steps.build.outcome == 'success' with: name: artifacts diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f2c9f8f5d..1c4527f02 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -38,14 +38,14 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: languages: c-cpp # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/autobuild@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index 41bfa222e..c01572a44 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -20,7 +20,7 @@ jobs: pull-requests: write steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 8ccc91e04..333546437 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index bab3a8264..82d365c6d 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -31,7 +31,7 @@ jobs: security-events: write steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -47,6 +47,6 @@ jobs: output: 'flawfinder_results.sarif' - name: Upload analysis results to GitHub Security tab - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: sarif_file: ${{github.workspace}}/flawfinder_results.sarif diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 3954235fc..21cf6c99c 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index dc13f42dc..979f2937f 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 1549f768e..a50c1d2bb 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -72,7 +72,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: SARIF file path: results.sarif @@ -80,6 +80,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: sarif_file: results.sarif diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 0eafc21d9..f9c11156d 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -65,7 +65,7 @@ jobs: # Upload SARIF file generated in previous step - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 with: sarif_file: semgrep.sarif if: always() diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 74e7c011f..10d527504 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 5216e60b2..025f58179 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -25,7 +25,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -38,7 +38,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Check that Meson and pkg-config offer the CMake options run: make check_build_options - name: Install Meson @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -92,14 +92,14 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build run: cmake --build build --target ci_infer - name: Archive Infer report if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: infer-report path: ${{ github.workspace }}/build/build_infer/infer-out/report.txt @@ -112,7 +112,7 @@ jobs: target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind] steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -123,7 +123,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -142,7 +142,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -168,7 +168,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -178,7 +178,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -197,7 +197,7 @@ jobs: - name: Build run: cmake --build build --target ci_test_coverage - name: Archive coverage report - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: code-coverage-report path: ${{ github.workspace }}/build/html @@ -244,7 +244,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On - name: Build @@ -262,7 +262,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -279,7 +279,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Disable filesystem support (std::filesystem is unusable) for clang 7 / 8 / 9 / 10 run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV" if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }} @@ -299,7 +299,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -319,7 +319,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build with libc++ @@ -346,7 +346,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -366,7 +366,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 with: cmakeVersion: 4.3.4 # Clang: the std library module is provided by libc++ (the image's libstdc++ @@ -392,7 +392,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -409,7 +409,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -423,7 +423,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DJSON_CI=On - name: Build @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -443,7 +443,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Run CMake run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja - name: Build @@ -456,7 +456,7 @@ jobs: target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid] steps: - name: Harden Runner - uses: step-security/harden-runner@351661ca32ac09a36dc5ee2d536e3128f2a3c8ed # v2.22.0 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -465,7 +465,7 @@ jobs: persist-credentials: false # the git-revision-date-localized plugin needs the history; a shallow clone makes the strict build fail fetch-depth: ${{ matrix.target == 'ci_test_build_documentation' && '0' || '1' }} - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 if: matrix.target == 'ci_test_documentation_mermaid' with: node-version: 24 @@ -484,7 +484,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Check that the job runs natively on arm64 run: uname -m && test "$(uname -m)" = aarch64 - name: Run CMake diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 0ac496124..bbc1ef48b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -90,7 +90,7 @@ jobs: with: persist-credentials: false - name: Get latest CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 - name: Set extra CXX_FLAGS for latest std_version # /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which # specializes std::tuple (newly diagnosed by the VS2026 v145 toolset) @@ -214,7 +214,7 @@ jobs: # import-std support. Its opt-in token is CMake-version-specific, so pin # CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt. - name: Get pinned CMake and ninja - uses: lukka/get-cmake@fffaaafeea488556c2c12dad60690008bc1caacb # v4.4.2 + uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4 with: cmakeVersion: 4.3.4 # CMAKE_CXX_FLAGS replaces CMake's MSVC defaults (which include /EHsc), and diff --git a/cmake/ci.cmake b/cmake/ci.cmake index cc8aaf620..d85214922 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -485,6 +485,13 @@ add_custom_target(ci_clang_analyze # Check code with Cppcheck. ############################################################################### +# default_defines.hpp expands the Hedley macros to the compiler's feature-test operators, which Cppcheck does not know. +# Since Cppcheck 2.20, it rejects their invocations in #if conditions instead of evaluating them as 0. +file(WRITE ${CMAKE_BINARY_DIR}/cppcheck_feature_tests.hpp "") +foreach(OPERATOR __has_attribute __has_builtin __has_cpp_attribute __has_declspec_attribute __has_extension __has_feature __has_include __has_warning) + file(APPEND ${CMAKE_BINARY_DIR}/cppcheck_feature_tests.hpp "#define ${OPERATOR}(x) 0\n") +endforeach() + add_custom_target(ci_cppcheck COMMAND mkdir -p cppcheck COMMAND clang -dM -E -x c++ -std=c++11 ${CMAKE_SOURCE_DIR}/include/nlohmann/thirdparty/hedley/hedley.hpp > default_defines.hpp 2> /dev/null @@ -493,7 +500,7 @@ add_custom_target(ci_cppcheck COMMAND venv_cppcheck/bin/cppcheck --enable=warning --check-level=exhaustive --inline-suppr --inconclusive --force --std=c++11 ${PROJECT_SOURCE_DIR}/include/nlohmann/json.hpp -I ${CMAKE_SOURCE_DIR}/include --error-exitcode=1 --relative-paths=${PROJECT_SOURCE_DIR} -j ${N} --include=default_defines.hpp - --cppcheck-build-dir=cppcheck + --include=${CMAKE_BINARY_DIR}/cppcheck_feature_tests.hpp --cppcheck-build-dir=cppcheck -UJSON_CATCH_USER -UJSON_TRY_USER -UJSON_ASSERT -UJSON_INTERNAL_CATCH -UJSON_THROW -DJSON_HAS_CPP_11 -UJSON_HAS_CPP_14 -UJSON_HAS_CPP_17 -UJSON_HAS_CPP_20 -UJSON_HAS_THREE_WAY_COMPARISON COMMENT "Check code with Cppcheck" diff --git a/cmake/requirements/requirements-cppcheck.txt b/cmake/requirements/requirements-cppcheck.txt index aa1591128..ecfb21ac7 100644 --- a/cmake/requirements/requirements-cppcheck.txt +++ b/cmake/requirements/requirements-cppcheck.txt @@ -1 +1 @@ -cppcheck==1.5.1 +cppcheck==1.5.2 diff --git a/docs/mkdocs/docs/api/basic_json/basic_json.md b/docs/mkdocs/docs/api/basic_json/basic_json.md index 78fa3560c..e553509ea 100644 --- a/docs/mkdocs/docs/api/basic_json/basic_json.md +++ b/docs/mkdocs/docs/api/basic_json/basic_json.md @@ -68,6 +68,14 @@ basic_json(basic_json&& other) noexcept; `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with a `key_type` compatible to `string_t` and a `value_type` from which a `basic_json` value can be constructed. - **strings**: `string_t`, string literals, and all compatible string containers can be used. + + !!! note "Rvalue containers are moved from" + + If `val` is an rvalue of a compatible array-like or object-like container other than `array_t`/`object_t` + (e.g., `json j = std::move(vec);` or `json j(std::vector{...});`), the elements are moved into + the JSON value instead of copied. Afterwards, the elements of the source container are in a valid but + unspecified state (**moved-from**). Keys of associative containers are `const` and therefore still copied. + Lvalues and `const` rvalues are copied, and the source is left unchanged. - **numbers**: [`number_integer_t`](number_integer_t.md), [`number_unsigned_t`](number_unsigned_t.md), [`number_float_t`](number_float_t.md), and all convertible number types such as `int`, `size_t`, `int64_t`, `float` or `double` can be used. @@ -236,7 +244,9 @@ basic_json(basic_json&& other) noexcept; 2. No-throw guarantee: this constructor never throws exceptions. 3. Depends on the called constructor. For types directly supported by the library (i.e., all types for which no `to_json()` function was provided), a strong guarantee holds: if an exception is thrown, there are no changes to any - JSON value. + JSON value. **Exception:** when `val` is an rvalue array-like or object-like container, its elements are moved one + by one; if an exception is thrown in the middle of the conversion (e.g., an allocation failure), the JSON value is + unchanged, but the source container may be left partially moved-from. 4. Depends on the called constructor. For types directly supported by the library (i.e., all types for which no `to_json()` function was provided), a strong guarantee holds: if an exception is thrown, there are no changes to any JSON value. diff --git a/docs/mkdocs/docs/api/basic_json/is_number_unsigned.md b/docs/mkdocs/docs/api/basic_json/is_number_unsigned.md index 50083164e..4dfb99049 100644 --- a/docs/mkdocs/docs/api/basic_json/is_number_unsigned.md +++ b/docs/mkdocs/docs/api/basic_json/is_number_unsigned.md @@ -6,6 +6,9 @@ constexpr bool is_number_unsigned() const noexcept; This function returns `#!cpp true` if and only if the JSON value is an unsigned integer number. This excludes floating-point and signed integer values. + +If [`number_unsigned_t`](number_unsigned_t.md) is the same type as [`number_integer_t`](number_integer_t.md), every +integer is stored as `number_integer_t`, and this function always returns `#!cpp false`. ## Return value diff --git a/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md b/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md index 217a59f5a..cd8044bd3 100644 --- a/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md +++ b/docs/mkdocs/docs/api/basic_json/number_unsigned_t.md @@ -25,7 +25,8 @@ the type to use. `NumberUnsignedType` : the type to store unsigned integers. It must be an **unsigned integral** type (`#!cpp std::is_integral`) with a `#!cpp std::numeric_limits` specialization, and it must be able to represent the absolute value of every - [`number_integer_t`](number_integer_t.md) value. See + [`number_integer_t`](number_integer_t.md) value. Alternatively, it can be the same type as + [`number_integer_t`](number_integer_t.md); then every integer is stored as `number_integer_t`. See [Template Parameter Requirements](../../features/types/template_parameters.md#numberintegertype-and-numberunsignedtype). ## Notes @@ -89,3 +90,4 @@ Integer number values are stored directly inside a `basic_json` type. ## Version history - Added in version 2.0.0. +- Can be the same type as `number_integer_t` since version 3.13.0. diff --git a/docs/mkdocs/docs/features/types/number_handling.md b/docs/mkdocs/docs/features/types/number_handling.md index b65708040..d11ad2747 100644 --- a/docs/mkdocs/docs/features/types/number_handling.md +++ b/docs/mkdocs/docs/features/types/number_handling.md @@ -345,8 +345,11 @@ The number types can be changed with template parameters. in case of overflow. - The type for unsigned integers must be convertible from `#!c unsigned long long`. The type for floating-point numbers is used in case of overflow. - - The types for signed and unsigned integers must be distinct, see - [#2573](https://github.com/nlohmann/json/issues/2573). + - The type for unsigned integers can also be the same signed type as the type for signed integers (e.g., + `#!c std::int64_t` for both). Then every integer is stored as a signed integer, and + [`is_number_unsigned()`](../../api/basic_json/is_number_unsigned.md) is always `#!c false`, see + [Template Parameter Requirements](template_parameters.md#same-type-for-both). + - The type for floating-point numbers must be distinct from the integer types. - Only `#!c double`, `#!c float`, and `#!c long double` are supported for floating-point numbers. !!! example diff --git a/docs/mkdocs/docs/features/types/template_parameters.md b/docs/mkdocs/docs/features/types/template_parameters.md index 162cfbad5..888590234 100644 --- a/docs/mkdocs/docs/features/types/template_parameters.md +++ b/docs/mkdocs/docs/features/types/template_parameters.md @@ -40,19 +40,19 @@ Requirements are split into two groups: ## Overview -| Template parameter | Default | Notable substitutes | -|-------------------------------------------------------------------|-----------------------------------|-----------------------------------------------------------------------| -| [`ObjectType`](#objecttype) | `std::map` | [`nlohmann::ordered_map`](../../api/ordered_map.md), Abseil hash maps | -| [`ArrayType`](#arraytype) | `std::vector` | `#!cpp std::deque` | -| [`StringType`](#stringtype) | `std::string` | `std::string`-like types over `char` | -| [`BooleanType`](#booleantype) | `bool` | none worth using | -| [`NumberIntegerType`](#numberintegertype-and-numberunsignedtype) | `std::int64_t` | any signed integer type | -| [`NumberUnsignedType`](#numberintegertype-and-numberunsignedtype) | `std::uint64_t` | any unsigned integer type at least as wide as `NumberIntegerType` | -| [`NumberFloatType`](#numberfloattype) | `double` | `float` (`long double`: no binary formats) | -| [`AllocatorType`](#allocatortype) | `std::allocator` | stateless allocators | -| [`JSONSerializer`](#jsonserializer) | `adl_serializer` | serializers with the same interface | -| [`BinaryType`](#binarytype) | `#!cpp std::vector` | `#!cpp std::vector` | -| [`CustomBaseClass`](#custombaseclass) | `void` | any default-constructible class | +| Template parameter | Default | Notable substitutes | +|-------------------------------------------------------------------|-----------------------------------|--------------------------------------------------------------------------------------------------| +| [`ObjectType`](#objecttype) | `std::map` | [`nlohmann::ordered_map`](../../api/ordered_map.md), Abseil hash maps | +| [`ArrayType`](#arraytype) | `std::vector` | `#!cpp std::deque` | +| [`StringType`](#stringtype) | `std::string` | `std::string`-like types over `char` | +| [`BooleanType`](#booleantype) | `bool` | none worth using | +| [`NumberIntegerType`](#numberintegertype-and-numberunsignedtype) | `std::int64_t` | any signed integer type | +| [`NumberUnsignedType`](#numberintegertype-and-numberunsignedtype) | `std::uint64_t` | any unsigned integer type at least as wide as `NumberIntegerType`, or `NumberIntegerType` itself | +| [`NumberFloatType`](#numberfloattype) | `double` | `float` (`long double`: no binary formats) | +| [`AllocatorType`](#allocatortype) | `std::allocator` | stateless allocators | +| [`JSONSerializer`](#jsonserializer) | `adl_serializer` | serializers with the same interface | +| [`BinaryType`](#binarytype) | `#!cpp std::vector` | `#!cpp std::vector` | +| [`CustomBaseClass`](#custombaseclass) | `void` | any default-constructible class | !!! warning "Third-party containers and incomplete types" @@ -504,13 +504,14 @@ Both types are stored **directly** inside `basic_json`'s union. ### Always required - `#!cpp std::is_integral` must be satisfied: `NumberIntegerType` must be a **signed** integer type, - `NumberUnsignedType` an **unsigned** integer type. Class types are not supported -- among others, the constructors - taking integer values are constrained on `#!cpp std::is_integral`. + `NumberUnsignedType` an **unsigned** integer type or the same type as `NumberIntegerType` (see + [below](#same-type-for-both)). Class types are not supported -- among others, the constructors taking integer values + are constrained on `#!cpp std::is_integral`. - Trivially default-constructible, trivially copyable, and trivially destructible (union member). - `#!cpp std::numeric_limits` must be specialized for both types. -- `NumberUnsignedType` must be able to represent the absolute value of every `NumberIntegerType` value; serialization - of negative numbers converts the value to `NumberUnsignedType`. A `#!cpp static_assert` requires it to be at least as - wide as `NumberIntegerType`, which is what that amounts to for the standard integer types. +- An unsigned `NumberUnsignedType` must be able to represent the absolute value of every `NumberIntegerType` value; + serialization of negative numbers converts the value to `NumberUnsignedType`. A `#!cpp static_assert` requires it to + be at least as wide as `NumberIntegerType`, which is what that amounts to for the standard integer types. - Both types must fit into the internal 64-character number buffer used by [`dump`](../../api/basic_json/dump.md), which is the case for all standard integer types. - [`std::hash`](../../api/basic_json/std_hash.md) additionally requires `#!cpp std::hash` specializations. @@ -522,6 +523,23 @@ is stored as [`number_float_t`](../../api/basic_json/number_float_t.md) instead. therefore silently changes parse results rather than raising an error. See [Number Handling](number_handling.md) for details. +#### Same type for both + +`NumberUnsignedType` may be the same type as `NumberIntegerType`, e.g. +`#!cpp nlohmann::json::with_integers_t` (see +[`with_integers_t`](../../api/basic_json/with_t.md)), for code that only wants to handle one integer type. Such a `basic_json` stores every integer as +[`number_integer_t`](../../api/basic_json/number_integer_t.md): + +- [`is_number_unsigned()`](../../api/basic_json/is_number_unsigned.md) is always `#!cpp false`, and + [`type()`](../../api/basic_json/type.md) never returns `value_t::number_unsigned`. Constructing a value from + `value_t::number_unsigned` gives a `value_t::number_integer` value. +- Values of unsigned C++ types (e.g., `#!cpp 5u`) are converted to `NumberIntegerType`, so values above its maximum + wrap around, as values outside the range of a narrower `NumberIntegerType` already do. +- The parsers store a non-negative integer above the maximum of `NumberIntegerType` (e.g., `18446744073709551615` for + `#!cpp std::int64_t`) as [`number_float_t`](../../api/basic_json/number_float_t.md), like any other integer that does + not fit. +- Converting from a `basic_json` with distinct types turns its unsigned values into `number_integer_t` values. + ### Compatible types | Type pair | Support | @@ -529,8 +547,9 @@ therefore silently changes parse results rather than raising an error. See | `#!cpp std::int64_t` / `#!cpp std::uint64_t` (default) | full | | `#!cpp std::int32_t` / `#!cpp std::uint32_t`, `#!cpp long long` / `#!cpp unsigned long long` | full; narrower types change which literals the parser can represent | | any other pair of standard signed/unsigned integer types | full | +| the same signed integer type for both, e.g. `#!cpp std::int64_t` / `#!cpp std::int64_t` | full; every integer is stored as `number_integer_t`, see [above](#same-type-for-both) | | class types, enumerations | not usable; `#!cpp std::is_integral` must hold | -| `#!cpp bool`, or a type already used for another member of the union | not usable; `#!cpp std::is_integral` is in fact `#!cpp true`, but the `get_impl_ptr` overloads for `boolean_t`, `number_integer_t`, `number_unsigned_t` and `number_float_t` would collide | +| `#!cpp bool`, `number_float_t`, or the type of another member of the union | not usable; `#!cpp std::is_integral` is in fact `#!cpp true`, but the `get_impl_ptr` overloads for `boolean_t`, `number_integer_t`, `number_unsigned_t` and `number_float_t` would collide | ## `NumberFloatType` diff --git a/docs/mkdocs/docs/home/releases.md b/docs/mkdocs/docs/home/releases.md index fabf95e8a..c0865270e 100644 --- a/docs/mkdocs/docs/home/releases.md +++ b/docs/mkdocs/docs/home/releases.md @@ -9,6 +9,15 @@ published on the [GitHub releases page](https://github.com/nlohmann/json/release This documentation is built from the `develop` branch and may describe changes that are not part of a release yet. Their version numbers are followed by an unreleased badge. +## Unreleased + +- Converting an rvalue container to JSON (`json j = std::move(vec);`, `json j = std::map{...};`) + now **moves** the elements instead of copying them (e.g., `std::vector` with 100,000 strings: 4.7 ms to + 2.3 ms). Two consequences: the source container is left in a valid but unspecified (moved-from) state, and if an + exception is thrown during the conversion, the source may be partially moved-from, so the strong exception guarantee + no longer holds for the source of an rvalue conversion (the JSON value itself is unaffected). Lvalues and `const` + objects are still copied. See [`basic_json`](../api/basic_json/basic_json.md). + ## v3.12.0 (2025-04-11) Fixes bugs found in 3.11.3 and adds several features. All changes are backward-compatible. diff --git a/include/nlohmann/detail/conversions/from_json.hpp b/include/nlohmann/detail/conversions/from_json.hpp index 0332474b2..5cb123c59 100644 --- a/include/nlohmann/detail/conversions/from_json.hpp +++ b/include/nlohmann/detail/conversions/from_json.hpp @@ -80,7 +80,11 @@ void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) { case value_t::number_unsigned: { - val = static_cast(*j.template get_ptr()); + // never null: a basic_json whose number_unsigned_t is number_integer_t + // stores no number_unsigned values (#2573) + const auto* ptr = j.template get_ptr(); + JSON_ASSERT(ptr != nullptr); + val = static_cast(*ptr); break; } case value_t::number_integer: @@ -150,7 +154,7 @@ inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_flo } template -inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +inline void from_json(const BasicJsonType& j, number_unsigned_overload_t& val) { get_arithmetic_value(j, val); } @@ -266,6 +270,22 @@ template void from_json_array_reserve(ConstructibleArrayType& /*arr*/, std::size_t /*size*/, priority_tag<0> /*unused*/) {} +// append through push_back() where available; containers without it (such as +// std::set) fall back to insert(end(), ...), which is what std::inserter did +template +auto from_json_array_append(ConstructibleArrayType& arr, ValueType&& value, priority_tag<1> /*unused*/) +-> decltype(arr.push_back(std::forward(value)), void()) +{ + arr.push_back(std::forward(value)); +} + +template +void from_json_array_append(ConstructibleArrayType& arr, ValueType&& value, priority_tag<0> /*unused*/) +{ + using std::end; + arr.insert(end(arr), std::forward(value)); +} + template::value, @@ -275,17 +295,15 @@ auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, p j.template get(), void()) { - using std::end; - ConstructibleArrayType ret; - from_json_array_reserve(ret, j.size(), priority_tag<1> {}); - std::transform(j.begin(), j.end(), - std::inserter(ret, end(ret)), [](const BasicJsonType & i) + const auto* inner_array = j.template get_ptr(); + from_json_array_reserve(ret, inner_array->size(), priority_tag<1> {}); + for (const auto& elem : *inner_array) { // get() returns *this, this won't call a from_json // method when value_type is BasicJsonType - return i.template get(); - }); + from_json_array_append(ret, elem.template get(), priority_tag<1> {}); + } arr = std::move(ret); } @@ -371,6 +389,23 @@ template inline void from_json_object_reserve(ConstructibleObjectType& /*obj*/, std::size_t /*size*/, priority_tag<0> /*unused*/) {} +// the keys of the source are sorted for the usual object types, so hinting the +// end makes every insertion constant-time (for containers that order their keys +// the same way); containers without emplace_hint() (such as ordered_map) fall +// back to emplace() +template +auto from_json_object_emplace(ConstructibleObjectType& obj, Key&& key, Value&& value, priority_tag<1> /*unused*/) +-> decltype(obj.emplace_hint(obj.end(), std::forward(key), std::forward(value)), void()) +{ + obj.emplace_hint(obj.end(), std::forward(key), std::forward(value)); +} + +template +void from_json_object_emplace(ConstructibleObjectType& obj, Key&& key, Value&& value, priority_tag<0> /*unused*/) +{ + obj.emplace(std::forward(key), std::forward(value)); +} + template::value, int> = 0> inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) @@ -385,7 +420,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) from_json_object_reserve(ret, inner_object->size(), priority_tag<1> {}); for (const auto& p : *inner_object) { - ret.emplace(p.first, p.second.template get()); + from_json_object_emplace(ret, p.first, p.second.template get(), priority_tag<1> {}); } obj = std::move(ret); } @@ -406,7 +441,11 @@ inline void from_json(const BasicJsonType& j, ArithmeticType& val) { case value_t::number_unsigned: { - val = static_cast(*j.template get_ptr()); + // never null: a basic_json whose number_unsigned_t is number_integer_t + // stores no number_unsigned values (#2573) + const auto* ptr = j.template get_ptr(); + JSON_ASSERT(ptr != nullptr); + val = static_cast(*ptr); break; } case value_t::number_integer: diff --git a/include/nlohmann/detail/conversions/to_json.hpp b/include/nlohmann/detail/conversions/to_json.hpp index d0d9610f4..5b217d7fe 100644 --- a/include/nlohmann/detail/conversions/to_json.hpp +++ b/include/nlohmann/detail/conversions/to_json.hpp @@ -167,6 +167,17 @@ struct external_constructor j.replace_value(value_t::array, value); } + /// an rvalue container: move its elements + template + static void construct_moving(BasicJsonType& j, CompatibleArrayType& arr) + { + using std::begin; + using std::end; + + const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(arr)), std::make_move_iterator(end(arr)))); + j.replace_value(value_t::array, value); + } + template static void construct(BasicJsonType& j, const std::vector& arr) { @@ -217,6 +228,17 @@ struct external_constructor j.replace_value(value_t::object, value); } + /// an rvalue container: move its values + template + static void construct_moving(BasicJsonType& j, CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(obj)), std::make_move_iterator(end(obj)))); + j.replace_value(value_t::object, value); + } + template < typename BasicJsonType, typename CompatibleObjectType, enable_if_t < !std::is_same::value, int > = 0 > static void construct(BasicJsonType& j, const CompatibleObjectType& obj) @@ -303,14 +325,14 @@ inline void to_json(BasicJsonType& j, FloatType val) noexcept } template::value, int> = 0> + enable_if_t, CompatibleNumberUnsignedType>::value, int> = 0> inline void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept { external_constructor::construct(j, static_cast(val)); } template::value, int> = 0> + enable_if_t::value, int> = 0> inline void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept { external_constructor::construct(j, static_cast(val)); @@ -354,6 +376,66 @@ inline void to_json(BasicJsonType& j, const CompatibleArrayType& arr) external_constructor::construct(j, arr); } +// types that have a dedicated to_json overload and must not be treated as generic containers +template +struct has_dedicated_to_json : std::false_type {}; + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template<> +struct has_dedicated_to_json : std::true_type {}; +#endif + +#if defined(__cpp_lib_char8_t) +template +struct has_dedicated_to_json> : std::true_type {}; +#endif + +#if JSON_HAS_RANGE_VIEW_CONVERSION +template +struct is_range_view_type : is_compatible_range_view {}; +#else +template +struct is_range_view_type : std::false_type {}; +#endif + +#if JSON_USE_OBJECTS_FOR_ENUM_KEYED_MAPS +template +struct is_enum_keyed_map_type : is_enum_keyed_map {}; +#else +template +struct is_enum_keyed_map_type : std::false_type {}; +#endif + +// rvalue (non-const) container: the generic to_json takes its elements by move +template +using is_movable_container_rvalue = negation, + std::is_const, + has_dedicated_to_json, + is_basic_json, + is_range_view_type, + is_enum_keyed_map_type, + std::is_same>>>; + +// rvalue array-like containers +template +using is_rvalue_array_to_json = conjunction, + is_compatible_array_type, + negation, + std::is_same, + is_compatible_object_type, + is_compatible_string_type, + is_compatible_binary_type>>>; + +// rvalue containers: their elements are moved instead of copied +template::value, int> = 0> +inline void to_json(BasicJsonType& j, CompatibleArrayType && arr) +{ + external_constructor::construct_moving(j, arr); +} + #if JSON_HAS_RANGE_VIEW_CONVERSION template < typename BasicJsonType, typename T, enable_if_t < is_compatible_range_view>::value @@ -427,6 +509,19 @@ inline void to_json(BasicJsonType& j, const EnumKeyedMap& map) } #endif +// rvalue object-like containers +template +using is_rvalue_object_to_json = conjunction, + negation>, + is_compatible_object_type>; + +template::value, int> = 0> +inline void to_json(BasicJsonType& j, CompatibleObjectType && obj) +{ + external_constructor::construct_moving(j, obj); +} + template inline void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) { diff --git a/include/nlohmann/detail/conversions/zmij.hpp b/include/nlohmann/detail/conversions/zmij.hpp index 40e477f7b..3820d891a 100644 --- a/include/nlohmann/detail/conversions/zmij.hpp +++ b/include/nlohmann/detail/conversions/zmij.hpp @@ -184,7 +184,10 @@ JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexc const uint128_parts p10 = pow10(-dec_exp - 1); const uint128_parts p = umul192_hi128(p10.high, p10.low, bin_sig << static_cast(shift)); std::uint64_t integral = p.high >> static_cast(extra_shift); - const std::uint64_t fractional = (p.high << static_cast(64 - extra_shift)) | (p.low >> static_cast(extra_shift)); + // (no casts in the shifts: MSVC 2015 warns C4554 on them despite the parentheses) + constexpr unsigned fractional_low_shift = static_cast(extra_shift); + constexpr unsigned fractional_high_shift = 64u - fractional_low_shift; + const std::uint64_t fractional = (p.high << fractional_high_shift) | (p.low >> fractional_low_shift); std::uint64_t digit = 0; bool round_up = false; bool round_down = false; diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index f2c6651c4..b2b5f36dd 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -428,7 +428,7 @@ void templated_json_throw(ExceptionType exception) @throw type_error.302 always */ template -JSON_HEDLEY_NO_RETURN inline void throw_type_must_be(const char* expected, const BasicJsonType& j) +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_type_must_be(const char* expected, const BasicJsonType& j) { static_cast(expected); // unused when JSON_NOEXCEPTION is defined static_cast(j); @@ -443,7 +443,7 @@ JSON_HEDLEY_NO_RETURN inline void throw_type_must_be(const char* expected, const @throw type_error always */ template -JSON_HEDLEY_NO_RETURN inline void throw_cannot_use_with(const exception_id id_, const char* operation, const BasicJsonType& j) +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_cannot_use_with(const exception_id id_, const char* operation, const BasicJsonType& j) { static_cast(id_); // unused when JSON_NOEXCEPTION is defined static_cast(operation); @@ -451,6 +451,77 @@ JSON_HEDLEY_NO_RETURN inline void throw_cannot_use_with(const exception_id id_, JSON_THROW(type_error::create(id_, concat("cannot use ", operation, " with ", j.type_name()), &j)); } +/*! +@brief throws ExceptionType with a fixed message; kept out of line so that callers do not carry the exception construction +*/ +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_literal(const exception_id id_, const char* what_arg, BasicJsonContext context) +{ + static_cast(id_); // unused when JSON_NOEXCEPTION is defined + static_cast(what_arg); + static_cast(context); + JSON_THROW(ExceptionType::create(id_, what_arg, context)); +} + +// JSON Patch errors; constructed out of line so that patch_inplace() does not carry them + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_add_parent(const BasicJsonType& parent) +{ + static_cast(parent); + JSON_THROW(out_of_range::create(exception_id::patch_add_parent_not_container, concat("cannot add value: the JSON Patch 'add' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_remove_parent(const BasicJsonType& parent) +{ + static_cast(parent); + JSON_THROW(out_of_range::create(exception_id::patch_remove_parent_not_container, concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_not_an_array(const BasicJsonType& j) +{ + static_cast(j); + JSON_THROW(parse_error::create(exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &j)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_missing_member(const StringType& op, const StringType& member, const bool string_type, const BasicJsonType& val) +{ + static_cast(op); + static_cast(member); + static_cast(string_type); + static_cast(val); + // context-sensitive error message + const std::string error_msg = (op == "op") ? std::string("operation") : concat("operation '", op, '\''); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat(error_msg, string_type ? " must have string member '" : " must have member '", member, "'"), &val)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_move_into_child(const StringType& from_path, const StringType& path, const BasicJsonType& result) +{ + static_cast(from_path); + static_cast(path); + static_cast(result); + JSON_THROW(out_of_range::create(exception_id::patch_move_into_child, concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_test_failed(const BasicJsonType& val) +{ + static_cast(val); + JSON_THROW(other_error::create(exception_id::patch_test_failed, concat("unsuccessful: ", val.dump()), &val)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_invalid_operation(const StringType& op, const BasicJsonType& val) +{ + static_cast(op); + static_cast(val); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat("operation value '", op, "' is invalid"), &val)); +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 4fdb31609..a584ee5e3 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -195,7 +195,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current != char_traits::eof())) { - return last_byte_error(exception_id::unexpected_end_of_input, concat("expected end of input; last byte: 0x", get_token_string()), "value"); + return last_byte_error(exception_id::unexpected_end_of_input, "expected end of input", "value", ""); } } @@ -317,7 +317,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) { - return last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"); + return bson_document_size_error(document_size, chars_read - document_start); } return true; } @@ -509,7 +509,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(len < 1)) { - return last_byte_error(exception_id::unexpected_byte, concat("string length must be at least 1, is ", std::to_string(len)), "string"); + return bson_length_error("string length must be at least 1, is ", len, "string"); } if (JSON_HEDLEY_UNLIKELY(!get_string(len - static_cast(1), result))) @@ -539,7 +539,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(len < 0)) { - return last_byte_error(exception_id::unexpected_byte, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"); + return bson_length_error("byte array length cannot be negative, is ", len, "binary"); } // All BSON binary values have a subtype @@ -629,9 +629,7 @@ class binary_reader default: // anything else is not supported (yet) { - const std::string cr_str = hex_byte(static_cast(element_type)); - return sax->parse_error(element_type_parse_position, cr_str, - parse_error::create(exception_id::bson_unsupported_type, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + return bson_unsupported_type_error(element_type_parse_position, element_type); } } } @@ -1060,6 +1058,7 @@ class binary_reader @param[in] context parsing context for the error message @return whether the SAX consumer accepts the parse error */ + JSON_HEDLEY_NEVER_INLINE bool cbor_indefinite_string_error(const char* type_name, const char* context) { return last_byte_error(exception_id::invalid_string_or_size, concat("indefinite-length ", type_name, " is not allowed inside indefinite-length ", type_name, "; last byte: 0x", get_token_string()), context); @@ -1139,7 +1138,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x60-0x7B)", inside_indefinite ? "" : " or indefinite string type (0x7F)", "; last byte: 0x", get_token_string()), "string"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x60-0x7B)", "string", inside_indefinite ? "" : " or indefinite string type (0x7F)"); } } } @@ -1269,7 +1268,7 @@ class binary_reader break; } - return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key"); + return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found); } /*! @@ -1350,7 +1349,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x40-0x5B)", inside_indefinite ? "" : " or indefinite binary array type (0x5F)", "; last byte: 0x", get_token_string()), "binary"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x40-0x5B)", "binary", inside_indefinite ? "" : " or indefinite binary array type (0x5F)"); } } } @@ -1496,8 +1495,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message(concat("excessive ", context, " size"), "size"), nullptr)); + return container_size_error(context); } result = conditional_static_cast(len); return true; @@ -2065,7 +2063,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", get_token_string()), "string"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0xA0-0xBF, 0xD9-0xDB)", "string", ""); } } } @@ -2157,7 +2155,7 @@ class binary_reader break; } - return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key"); + return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found); } /*! @@ -2655,8 +2653,7 @@ class binary_reader } if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(number))) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("integer value overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char return true; @@ -2753,8 +2750,7 @@ class binary_reader } if (!value_in_range_of(number)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("integer value overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } result = detail::conditional_static_cast(number); return true; @@ -2832,7 +2828,7 @@ class binary_reader // or SIZE_MAX. if (JSON_HEDLEY_UNLIKELY(result > (std::numeric_limits::max)() / i)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size"); } result *= i; // the pre-check above already rules out result becoming 0 @@ -2841,7 +2837,7 @@ class binary_reader // unknown-size container (see get_ubjson_size_type()) if (result == npos) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size"); } if (JSON_HEDLEY_UNLIKELY(!emit_unsigned(i))) { @@ -2891,7 +2887,7 @@ class binary_reader if (input_format == input_format_t::bjdata && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) { - return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type"); + return optimized_type_error(); } if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) @@ -2906,7 +2902,7 @@ class binary_reader { return false; } - return last_byte_error(exception_id::unexpected_byte, concat("expected '#' after type information; last byte: 0x", get_token_string()), "size"); + return last_byte_error(exception_id::unexpected_byte, "expected '#' after type information", "size", ""); } const bool is_error = get_ubjson_size_value(result.first, is_ndarray, 0, result.second); @@ -3057,7 +3053,7 @@ class binary_reader } if (JSON_HEDLEY_UNLIKELY(current > 127)) { - return last_byte_error(exception_id::invalid_string_or_size, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", get_token_string()), "char"); + return last_byte_error(exception_id::invalid_string_or_size, "byte after 'C' must be in range 0x00..0x7F", "char", ""); } string_t s(1, static_cast(current)); return sax->string(s); @@ -3141,8 +3137,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') && size_and_type.first > max_valueless_container_size)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("excessive array size", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive array size", "size"); } if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) @@ -3854,7 +3849,7 @@ class binary_reader { // in case of failure, advance position by 1 to report the failing location ++chars_read; - sax->parse_error(chars_read, "", parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + static_cast(report_eof(context)); return false; } return true; @@ -4007,8 +4002,7 @@ class binary_reader const auto result = static_cast(number); if (JSON_HEDLEY_UNLIKELY(std::isfinite(number) && !std::isfinite(result))) { - return sax->parse_error(chars_read, get_token_string(), - out_of_range::create(exception_id::number_overflow, exception_message("number overflow", "value"), nullptr)); + return out_of_range_error(exception_id::number_overflow, "number overflow", "value"); } return sax->number_float(result, ""); } @@ -4223,22 +4217,82 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(current == char_traits::eof())) { - return sax->parse_error(chars_read, "", - parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + return report_eof(context); } return true; } + /*! + @brief reports an unexpected end of input (out of line to keep the exception + construction out of the hot reading functions) + @param[in] context further context information (for diagnostics) + @return the result of the SAX parser's parse_error() + */ + JSON_HEDLEY_NON_NULL(2) + JSON_HEDLEY_NEVER_INLINE + bool report_eof(const char* context) const + { + return sax->parse_error(chars_read, "", + parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + } + + /*! + @brief reports an out_of_range error at the last read byte (out of line) + */ + JSON_HEDLEY_NEVER_INLINE + bool out_of_range_error(const exception_id id_, const char* detail, const char* context) const + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(id_, exception_message(detail, context), nullptr)); + } + + JSON_HEDLEY_NEVER_INLINE + bool container_size_error(const char* what) const + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, + exception_message(concat("excessive ", what, " size"), "size"), nullptr)); + } + + JSON_HEDLEY_NEVER_INLINE + bool optimized_type_error() const + { + return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type"); + } + + template + JSON_HEDLEY_NEVER_INLINE + bool bson_length_error(const char* prefix, const NumberType len, const char* context) const + { + return last_byte_error(exception_id::unexpected_byte, concat(prefix, std::to_string(len)), context); + } + + JSON_HEDLEY_NEVER_INLINE + bool bson_document_size_error(const std::int32_t document_size, const std::size_t bytes_read) const + { + return last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(bytes_read), ")"), "document"); + } + + JSON_HEDLEY_NEVER_INLINE + bool bson_unsupported_type_error(const std::size_t position, const char element_type) const + { + const std::string cr_str = hex_byte(static_cast(element_type)); + return sax->parse_error(position, cr_str, + parse_error::create(exception_id::bson_unsupported_type, position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + } + /*! @brief reports a parse error at the last read byte @param[in] id_ the id of the parse_error exception @param[in] detail a detailed error message @param[in] context further context information + @param[in] tail optional text appended to the detail, followed by "; last byte: 0x" @return the result of the SAX parser's parse_error() */ - bool last_byte_error(const exception_id id_, const std::string& detail, const std::string& context) const + JSON_HEDLEY_NEVER_INLINE + bool last_byte_error(const exception_id id_, const std::string& detail, const char* context, const char* tail = nullptr) const { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, exception_message(detail, context), nullptr)); + // with a tail, the message reads "; last byte: 0x" + return sax->parse_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, + exception_message(tail == nullptr ? detail : concat(detail, tail, "; last byte: 0x", get_token_string()), context), nullptr)); } /*! @@ -4247,7 +4301,8 @@ class binary_reader @param[in] context further context information @return the result of the SAX parser's parse_error() */ - bool unexpected_byte(const char* detail, const std::string& context) const + JSON_HEDLEY_NEVER_INLINE + bool unexpected_byte(const char* detail, const char* context) const { return last_byte_error(exception_id::unexpected_byte, concat(detail, ": 0x", get_token_string()), context); } @@ -4258,6 +4313,7 @@ class binary_reader @param[in] context further context information @return the result of the SAX parser's parse_error() */ + JSON_HEDLEY_NEVER_INLINE bool length_type_error(const char* position, const char* context) const { const char* types = input_format == input_format_t::bjdata ? "U, i, u, I, m, l, M, L" : "U, i, I, l, L"; diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 161568ded..912888324 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -187,12 +187,19 @@ Shared by json_sax_dom_parser and json_sax_dom_callback_parser. @param[in] ref the object or array that was just created @return whether parsing should continue (false after reporting out_of_range.408) */ +template +JSON_HEDLEY_NEVER_INLINE +bool container_too_large_error(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref) +{ + return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref)); +} + template bool check_container_size(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref) { if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref->max_size())) { - return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref)); + return container_too_large_error(sax, len, kind, ref); } return true; } @@ -674,16 +681,19 @@ class json_sax_dom_callback_parser if (keep && ref_stack.back()) { auto& obj = *ref_stack.back()->m_data.m_value.object; - const auto it = obj.find(val); - if (it != obj.end()) + // a single lookup: insert the placeholder, or find the existing member + const auto inserted = obj.emplace(val, discarded); + const auto it = inserted.first; + if (!inserted.second) { // this is a duplicate key (legal in JSON); remember its // current value so it can be restored later if the new // value is rejected by the callback, instead of being // erased together with the discarded placeholder duplicate_key_stash.emplace_back(&(it->second), it->second); + it->second = discarded; } - object_element = &(obj[val] = discarded); + object_element = &(it->second); } return true; diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index bb845cc1a..588db96a3 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -19,7 +19,7 @@ #include // memcpy #include // numeric_limits #include // string -#include // conditional, integral_constant, true_type, false_type +#include // conditional, integral_constant, is_signed, true_type, false_type #include // move #include @@ -50,6 +50,20 @@ NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { +// whether an integer of a signed type is negative; the overload for other types +// avoids comparing an unsigned value with zero +template +constexpr bool is_negative_integer(NumberType x, std::true_type /*is_signed*/) noexcept +{ + return x < 0; +} + +template +constexpr bool is_negative_integer(NumberType /*unused*/, std::false_type /*is_signed*/) noexcept +{ + return false; +} + /*! @brief fast integer parser for an already-validated unsigned integer @@ -83,8 +97,10 @@ bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedT x = (x * 10u) + digit; } value = static_cast(x); - // reject values that do not round-trip into a narrower NumberUnsignedType - return static_cast(value) == x; + // reject values that do not round-trip into a narrower NumberUnsignedType, + // and values that wrap around to negative numbers if it is a signed type + // (number_unsigned_t may be the same type as number_integer_t, #2573) + return static_cast(value) == x && !is_negative_integer(value, std::is_signed()); } /*! @@ -744,7 +760,7 @@ inline void float_from_bits(std::uint64_t bits, float& value) noexcept { static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits"); const auto bits32 = static_cast(bits); - std::memcpy(&value, &bits32, sizeof(value)); + std::memcpy(&value, &bits32, sizeof(bits32)); } /// the powers of ten that are exact in binary64 (up to 10^22) diff --git a/include/nlohmann/detail/iterators/iter_impl.hpp b/include/nlohmann/detail/iterators/iter_impl.hpp index 52b459ad2..b7c1df24a 100644 --- a/include/nlohmann/detail/iterators/iter_impl.hpp +++ b/include/nlohmann/detail/iterators/iter_impl.hpp @@ -280,7 +280,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci } /// @throw invalid_iterator.214 always - JSON_HEDLEY_NO_RETURN void throw_cannot_get_value() const + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_cannot_get_value() const { JSON_THROW(invalid_iterator::create(exception_id::iterator_value_unavailable, "cannot get value", m_object)); } @@ -294,40 +294,25 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); - return m_it.object_iterator->second; - } - - case value_t::array: - { - JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); - return *m_it.array_iterator; - } - - case value_t::null: - throw_cannot_get_value(); - - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) - { - return *m_object; - } - - throw_cannot_get_value(); - } + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); + return *m_it.array_iterator; } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); + return m_it.object_iterator->second; + } + + // neither array nor object: a primitive value (null has no value to point to) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type != value_t::null && m_it.primitive_iterator.is_begin())) + { + return *m_object; + } + + throw_cannot_get_value(); } /*! @@ -338,38 +323,25 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); - return &(m_it.object_iterator->second); - } - - case value_t::array: - { - JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); - return &*m_it.array_iterator; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) - { - return m_object; - } - - throw_cannot_get_value(); - } + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); + return &*m_it.array_iterator; } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); + return &(m_it.object_iterator->second); + } + + // neither array nor object: a primitive value + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return m_object; + } + + throw_cannot_get_value(); } /*! @@ -391,35 +363,20 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - std::advance(m_it.object_iterator, 1); - break; - } - - case value_t::array: - { - std::advance(m_it.array_iterator, 1); - break; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - ++m_it.primitive_iterator; - break; - } + ++m_it.array_iterator; + return *this; } + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + std::advance(m_it.object_iterator, 1); + return *this; + } + + // neither array nor object: a primitive value + ++m_it.primitive_iterator; return *this; } @@ -442,35 +399,20 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - std::advance(m_it.object_iterator, -1); - break; - } - - case value_t::array: - { - std::advance(m_it.array_iterator, -1); - break; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - --m_it.primitive_iterator; - break; - } + --m_it.array_iterator; + return *this; } + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + std::advance(m_it.object_iterator, -1); + return *this; + } + + // neither array nor object: a primitive value + --m_it.primitive_iterator; return *this; } @@ -484,7 +426,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci // if objects are not the same, the comparison is undefined if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) { - JSON_THROW(invalid_iterator::create(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object)); + throw_literal(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object); } // value-initialized forward iterators can be compared, and must compare equal to other value-initialized iterators of the same type #4493 @@ -493,25 +435,18 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci return true; } - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - return (m_it.object_iterator == other.m_it.object_iterator); - - case value_t::array: - return (m_it.array_iterator == other.m_it.array_iterator); - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - return (m_it.primitive_iterator == other.m_it.primitive_iterator); + return (m_it.array_iterator == other.m_it.array_iterator); } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + // neither array nor object: a primitive value + return (m_it.primitive_iterator == other.m_it.primitive_iterator); } /*! @@ -533,7 +468,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci // if objects are not the same, the comparison is undefined if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) { - JSON_THROW(invalid_iterator::create(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object)); + throw_literal(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object); } // value-initialized forward iterators can be compared, and must compare equal to other value-initialized iterators of the same type #4493 @@ -546,7 +481,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_order_on_object, "cannot compare order of object iterators", m_object)); + throw_literal(exception_id::iterator_order_on_object, "cannot compare order of object iterators", m_object); case value_t::array: return (m_it.array_iterator < other.m_it.array_iterator); @@ -602,7 +537,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object)); + throw_literal(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object); case value_t::array: { @@ -681,7 +616,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object)); + throw_literal(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object); case value_t::array: return m_it.array_iterator - other.m_it.array_iterator; @@ -710,7 +645,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_subscript_on_object, "cannot use operator[] for object iterators", m_object)); + throw_literal(exception_id::iterator_subscript_on_object, "cannot use operator[] for object iterators", m_object); case value_t::array: return *std::next(m_it.array_iterator, n); @@ -750,7 +685,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci return m_it.object_iterator->first; } - JSON_THROW(invalid_iterator::create(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object)); + throw_literal(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object); } /*! diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 1b439f9bf..a752deb08 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -16,14 +16,13 @@ #include // ostream #endif // JSON_NO_IO #include // max -#include // map -#include // accumulate #include // string #include // move #include // vector #include #include +#include #include #include #include @@ -69,12 +68,21 @@ class json_pointer /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ string_t to_string() const { - return std::accumulate(reference_tokens.begin(), reference_tokens.end(), - string_t{}, - [](const string_t& a, const string_t& b) + // one buffer: every token adds its escaped form (at least its length) and a slash + std::size_t length = 0; + for (const auto& reference_token : reference_tokens) { - return detail::concat(a, '/', detail::escape(b)); - }); + length += reference_token.size() + 1; + } + + string_t result; + result.reserve(length); + for (const auto& reference_token : reference_tokens) + { + result.append("/", 1); + detail::escape_into(result, reference_token); + } + return result; } /// @brief return a string representation of the JSON pointer @@ -129,21 +137,29 @@ class json_pointer friend json_pointer operator/(const json_pointer& lhs, const json_pointer& rhs) { - return json_pointer(lhs) /= rhs; + json_pointer result; + result.reference_tokens.reserve(lhs.reference_tokens.size() + rhs.reference_tokens.size()); + result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); + result.reference_tokens.insert(result.reference_tokens.end(), rhs.reference_tokens.begin(), rhs.reference_tokens.end()); + return result; } /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ friend json_pointer operator/(const json_pointer& lhs, string_t token) // NOLINT(performance-unnecessary-value-param) { - return json_pointer(lhs) /= std::move(token); + json_pointer result; + result.reference_tokens.reserve(lhs.reference_tokens.size() + 1); + result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); + result.reference_tokens.push_back(std::move(token)); + return result; } /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx) { - return json_pointer(lhs) /= array_idx; + return lhs / detail::to_string(array_idx); } /// @brief returns the parent of this JSON pointer @@ -155,8 +171,8 @@ class json_pointer return *this; } - json_pointer res = *this; - res.pop_back(); + json_pointer res; + res.reference_tokens = std::vector(reference_tokens.begin(), reference_tokens.end() - 1); return res; } @@ -245,14 +261,14 @@ class json_pointer private: /// @throw out_of_range.405 always - JSON_HEDLEY_NO_RETURN static void throw_no_parent() + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_no_parent() { JSON_THROW(detail::out_of_range::create(detail::exception_id::patch_on_root, "JSON pointer has no parent", nullptr)); } /// @throw out_of_range.404 always template - JSON_HEDLEY_NO_RETURN static void throw_unresolved(const string_t& reference_token, BasicJsonContext context) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_unresolved(const string_t& reference_token, BasicJsonContext context) { static_cast(reference_token); // unused when JSON_NOEXCEPTION is defined static_cast(context); @@ -305,6 +321,28 @@ class json_pointer return array_index_status::not_a_number; } + // fast path: up to 18 decimal digits cannot overflow unsigned long long or + // size_type (9 digits for a 32-bit size_type), so strtoull and errno are not needed + constexpr std::size_t max_fast_digits = sizeof(size_type) >= 8 ? 18 : 9; + if (!s.empty() && s.size() <= max_fast_digits) + { + size_type acc = 0; + for (std::size_t i = 0; i < s.size(); ++i) + { + const char c = s[i]; + if (JSON_HEDLEY_UNLIKELY(c < '0' || c > '9')) + { + break; // not a plain number: let strtoull decide + } + acc = static_cast(acc * 10 + static_cast(c - '0')); + if (JSON_HEDLEY_UNLIKELY(i == s.size() - 1)) + { + idx = acc; + return array_index_status::ok; + } + } + } + const char* p = s.data(); char* p_end = nullptr; // NOLINT(misc-const-correctness) errno = 0; // strtoull doesn't reset errno @@ -361,6 +399,83 @@ class json_pointer return idx; } + /*! + @brief compare with a JSON pointer string without splitting it into tokens + + @throw parse_error.107 if the string is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ + bool equals_string(const string_t& reference_string) const + { + if (reference_string.empty()) + { + return reference_tokens.empty(); + } + + if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/')) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr)); + } + + // the whole string is validated, whether or not it matches + const std::size_t size = reference_string.size(); + for (std::size_t pos = reference_string.find_first_of('~'); + pos != string_t::npos; + pos = reference_string.find_first_of('~', pos + 1)) + { + if (JSON_HEDLEY_UNLIKELY(pos == size - 1 || (reference_string[pos + 1] != '0' && reference_string[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + } + } + + // compare token by token, unescaping the string on the fly + std::size_t pos = 0; // the slash in front of the next token + for (const auto& reference_token : reference_tokens) + { + if (pos >= size) + { + return false; + } + + std::size_t i = pos + 1; + std::size_t j = 0; + const std::size_t token_size = reference_token.size(); + while (i < size && reference_string[i] != '/') + { + char c = reference_string[i++]; + if (c == '~') + { + c = (reference_string[i++] == '0') ? '~' : '/'; + } + if (j == token_size || reference_token[j] != c) + { + return false; + } + ++j; + } + if (j != token_size) + { + return false; + } + pos = i; + } + return pos == size; + } + + /// @brief object member access with a key that is only copied if it is inserted + template + static BasicJsonType& object_member(BasicJsonType& object, const string_t& key) + { + auto& members = *object.m_data.m_value.object; + const auto it = members.find(key); + if (it != members.end()) + { + return it->second; + } + return object.operator[](key); + } + JSON_PRIVATE_UNLESS_TESTED: json_pointer top() const { @@ -384,84 +499,141 @@ class json_pointer the result independent of the order in which the flattened object is iterated, which is unspecified for some object types. - The prefixes form a tree and are numbered, so each of them is stored only - once (as a node) rather than as a copy of all of its reference tokens. + The prefixes form a tree and are numbered (number 0 is the empty prefix), + so each of them is stored only once (as a node) rather than as a copy of + all of its reference tokens. The tree does not own the reference tokens: + it refers to the tokens of the parsed keys, which must outlive it. */ struct prefix_tree { - // children[id] maps a reference token to the number of the prefix - // extended by that token; number 0 is the empty prefix - std::vector> children; + // open-addressing hash table from (parent, last reference token) to + // the number of the node; empty slots have token == nullptr + struct slot + { + std::size_t hash; + std::size_t parent; + const string_t* token; + std::size_t id; + + slot() noexcept + : hash(0), parent(0), token(nullptr), id(0) + {} + + slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept + : hash(h), parent(p), token(t), id(i) + {} + }; + + std::vector slots; + std::size_t used = 0; + + static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept + { + std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL); + for (std::size_t i = 0; i < token.size(); ++i) + { + h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL); + } + return h ^ (h >> 29u); + } + + void grow() + { + std::vector old(slots.size() * 2); + old.swap(slots); + const std::size_t mask = slots.size() - 1; + for (const auto& s : old) + { + if (s.token != nullptr) + { + std::size_t i = s.hash & mask; + while (slots[i].token != nullptr) + { + i = (i + 1) & mask; + } + slots[i] = s; + } + } + } + + // parent[id] is the number of the prefix without its last token + std::vector parent; // is_array[id] is true iff some flattened key has the reference token // 0 directly below the prefix with number id std::vector is_array; // start with the empty prefix only prefix_tree() - : children(1) + : slots(16) + , parent(1, 0) , is_array(1, false) {} // return the number of the prefix with number id extended by // reference_token, adding it if it is new - std::size_t add_child(std::size_t id, string_t&& reference_token) + std::size_t add_child(std::size_t id, const string_t& reference_token) { + const std::size_t hash = hash_of(id, reference_token); + const std::size_t mask = slots.size() - 1; + std::size_t i = hash & mask; + while (slots[i].token != nullptr) + { + if (slots[i].hash == hash && slots[i].parent == id && *slots[i].token == reference_token) + { + return slots[i].id; + } + i = (i + 1) & mask; + } + + // new prefix + const std::size_t child = parent.size(); + slots[i] = slot(hash, id, &reference_token, child); if (reference_token == "0") { is_array[id] = true; } + parent.push_back(id); + is_array.push_back(false); - // read the number before the emplace_back below, which may - // reallocate children and invalidate the iterator - const std::size_t next = children.size(); - const auto inserted = children[id].emplace(std::move(reference_token), next); - const std::size_t child = inserted.first->second; - if (inserted.second) + // keep the load factor below 1/2 + if (++used * 2 > slots.size()) { - children.emplace_back(); - is_array.push_back(false); + grow(); } return child; } - - // return the number of the prefix with number id extended by - // reference_token, which must have been added before - std::size_t find_child(std::size_t id, const string_t& reference_token) const - { - const auto it = children[id].find(reference_token); - JSON_ASSERT(it != children[id].end()); - return it->second; - } }; /*! @brief create and return a reference to the pointed to value - Complexity: Linear in the number of reference tokens (times the logarithm - of the number of siblings for the prefix lookup). + @param[in] j the value to create the pointed to value in + @param[in] tokens the reference tokens of the pointer + @param[in] prefixes prefixes[i] is the number in @a tree of the prefix + consisting of the first i reference tokens + + Complexity: Linear in the number of reference tokens. @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ template - BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const + static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector& tokens, + const std::vector& prefixes, const prefix_tree& tree) { auto* result = &j; - // the number of the prefix consumed so far; used to look up whether - // the value to be created below is an array or an object - std::size_t id = 0; - // in case no reference tokens exist, return a reference to the JSON value // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) + for (std::size_t i = 0; i < tokens.size(); ++i) { + const auto& reference_token = tokens[i]; switch (result->type()) { case detail::value_t::null: { - if (tree.is_array[id]) + if (tree.is_array[prefixes[i]]) { // some reference token below this position is 0, so the // value is an array @@ -478,7 +650,7 @@ class json_pointer case detail::value_t::object: { // create an entry in the object - result = &result->operator[](reference_token); + result = &object_member(*result, reference_token); break; } @@ -505,8 +677,6 @@ class json_pointer default: JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j)); } - - id = tree.find_child(id, reference_token); } return *result; @@ -558,7 +728,7 @@ class json_pointer case detail::value_t::object: { // use unchecked object access - ptr = &ptr->operator[](reference_token); + ptr = &object_member(*ptr, reference_token); break; } @@ -608,8 +778,13 @@ class json_pointer { case detail::value_t::object: { - // note: at performs range check - ptr = &ptr->at(reference_token); + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) + { + // at() throws out_of_range.403 with the usual message + ptr->at(reference_token); + } + ptr = &it->second; break; } @@ -630,7 +805,7 @@ class json_pointer JSON_THROW(detail::out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat( "array index ", std::to_string(idx), " is out of range"), ptr)); } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -678,7 +853,9 @@ class json_pointer { // use unchecked object access; the const operator[] // asserts that the key exists - ptr = &ptr->operator[](reference_token); + const auto it = ptr->m_data.m_value.object->find(reference_token); + JSON_ASSERT(it != ptr->m_data.m_value.object->end()); + ptr = &it->second; break; } @@ -692,7 +869,9 @@ class json_pointer // use unchecked array access; the const operator[] // asserts that the index exists - ptr = &ptr->operator[](array_index(reference_token)); + const auto idx = array_index(reference_token); + JSON_ASSERT(idx < ptr->m_data.m_value.array->size()); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -733,12 +912,12 @@ class json_pointer { case detail::value_t::object: { - const auto it = ptr->find(reference_token); - if (JSON_HEDLEY_UNLIKELY(it == ptr->end())) + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) { return nullptr; } - ptr = &*it; + ptr = &it->second; break; } @@ -775,7 +954,7 @@ class json_pointer { return nullptr; } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -808,13 +987,14 @@ class json_pointer { case detail::value_t::object: { - if (!ptr->contains(reference_token)) + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (it == ptr->m_data.m_value.object->end()) { // we did not find the key in the object return false; } - ptr = &ptr->operator[](reference_token); + ptr = &it->second; break; } @@ -836,13 +1016,13 @@ class json_pointer return false; } - if (idx >= ptr->size()) + if (idx >= ptr->m_data.m_value.array->size()) { // index out of range return false; } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -892,6 +1072,15 @@ class json_pointer JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr)); } + // every slash starts a reference token + const std::size_t size = reference_string.size(); + std::size_t token_count = 0; + for (std::size_t i = 0; i < size; ++i) + { + token_count += (reference_string[i] == '/') ? 1 : 0; + } + result.reserve(token_count); + // extract the reference tokens: // - slash: position of the last read slash (or end of string) // - start: position after the previous slash @@ -913,31 +1102,52 @@ class json_pointer const auto count = (slash == string_t::npos ? reference_string.size() : slash) - start; auto reference_token = string_t(reference_string.data() + start, count); - // check reference tokens are properly escaped - for (std::size_t pos = reference_token.find_first_of('~'); - pos != string_t::npos; - pos = reference_token.find_first_of('~', pos + 1)) + // check reference tokens are properly escaped; a token without + // a '~' (the common case) needs neither check nor unescaping + std::size_t pos = reference_token.find_first_of('~'); + if (pos != string_t::npos) { - JSON_ASSERT(reference_token[pos] == '~'); - - // ~ must be followed by 0 or 1 - if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || - (reference_token[pos + 1] != '0' && - reference_token[pos + 1] != '1'))) + for (; pos != string_t::npos; pos = reference_token.find_first_of('~', pos + 1)) { - JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + JSON_ASSERT(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || + (reference_token[pos + 1] != '0' && + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + } } + + detail::unescape(reference_token); } // finally, store the reference token - detail::unescape(reference_token); - result.push_back(reference_token); + result.push_back(std::move(reference_token)); } return result; } private: + // Add the entry (key, v) to the object `obj`. Flattened paths are pairwise + // distinct, so objects that can append a key known to be absent + // (ordered_map) skip the lookup, which is linear and would make flatten + // quadratic; all other object types use operator[]. + template + static auto add_flattened(ObjectType& obj, const string_t& key, ValueType&& v, detail::priority_tag<1> /*unused*/) + -> decltype(obj.emplace_new_key(key, std::forward(v)), void()) + { + obj.emplace_new_key(key, std::forward(v)); + } + + template + static void add_flattened(ObjectType& obj, const string_t& key, ValueType&& v, detail::priority_tag<0> /*unused*/) + { + obj[key] = std::forward(v); + } + /*! @param[in] reference_string the reference string to the current value @param[in] value the value to consider @@ -982,7 +1192,8 @@ class json_pointer // flatten `v`, whose path is `path`: primitives and empty containers // are added to the result right away; other containers get a frame - const auto enter = [&stack, &path, &result](const BasicJsonType & v) + auto& object = *result.m_data.m_value.object; + const auto enter = [&stack, &path, &object](const BasicJsonType & v) { switch (v.type()) { @@ -991,7 +1202,7 @@ class json_pointer if (v.m_data.m_value.array->empty()) { // flatten empty array as null - result[path] = nullptr; + add_flattened(object, path, nullptr, detail::priority_tag<1> {}); } else { @@ -1005,7 +1216,7 @@ class json_pointer if (v.m_data.m_value.object->empty()) { // flatten empty object as null - result[path] = nullptr; + add_flattened(object, path, nullptr, detail::priority_tag<1> {}); } else { @@ -1025,7 +1236,7 @@ class json_pointer default: { // add a primitive value with its reference string - result[path] = v; + add_flattened(object, path, v, detail::priority_tag<1> {}); return; } } @@ -1071,6 +1282,10 @@ class json_pointer enter(it->second); } } + + // the entries were added without going through basic_json, so the + // parent pointers (JSON_DIAGNOSTICS) have to be set now + result.set_parents(); } /*! @@ -1094,21 +1309,53 @@ class json_pointer BasicJsonType result; + // parse every key once; the reference tokens are kept for the second + // pass (the vectors are never resized, so the tree may refer to them) + std::vector> keys; + keys.reserve(value.m_data.m_value.object->size()); + for (const auto& element : *value.m_data.m_value.object) + { + json_pointer ptr(element.first); + keys.push_back(std::move(ptr.reference_tokens)); + } + // collect the pointer prefixes that have a reference token 0 among // their children; the values below them are arrays, all others are // objects (see prefix_tree) prefix_tree tree; - for (const auto& element : *value.m_data.m_value.object) + // the number of the prefix with i + 1 tokens of the previous key; keys + // are often sorted, so most of the prefix is the same as for the + // previous key and need not be looked up again + std::vector path; + const std::vector* previous = nullptr; + // the number of the complete prefix (i.e., the key) of each key + std::vector leaves; + leaves.reserve(keys.size()); + for (const auto& tokens : keys) { - json_pointer ptr(element.first); - std::size_t id = 0; - for (auto& reference_token : ptr.reference_tokens) + std::size_t common = 0; + if (previous != nullptr) { - id = tree.add_child(id, std::move(reference_token)); + const std::size_t limit = (std::min)(tokens.size(), previous->size()); + while (common < limit && tokens[common] == (*previous)[common]) + { + ++common; + } } + path.resize(common); + std::size_t id = common == 0 ? 0 : path[common - 1]; + for (std::size_t i = common; i < tokens.size(); ++i) + { + id = tree.add_child(id, tokens[i]); + path.push_back(id); + } + leaves.push_back(id); + previous = &tokens; } // iterate the JSON object values + std::vector prefixes; + std::size_t key_index = 0; for (const auto& element : *value.m_data.m_value.object) { if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) @@ -1116,11 +1363,21 @@ class json_pointer JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second)); } + const auto& tokens = keys[key_index]; + // number the prefixes of this key from its last one to the first + prefixes.resize(tokens.size()); + for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i) + { + id = tree.parent[id]; + prefixes[i - 1] = id; + } + ++key_index; + // Assign the value to the reference pointed to by JSON pointer. Note // that if the JSON pointer is "" (i.e., points to the whole value), // function get_and_create returns a reference to the result itself. // An assignment will then create a primitive value. - json_pointer(element.first).get_and_create(result, tree) = element.second; + get_and_create(result, tokens, prefixes, tree) = element.second; } return result; @@ -1159,7 +1416,7 @@ class json_pointer = delete; #else { - return *this == json_pointer(rhs); + return equals_string(rhs); } #endif @@ -1243,7 +1500,7 @@ inline bool operator==(const json_pointer& lhs, = delete; #else { - return lhs == json_pointer(rhs); + return lhs.equals_string(rhs); } #endif @@ -1256,7 +1513,7 @@ inline bool operator==(const StringType& lhs, = delete; #else { - return json_pointer(lhs) == rhs; + return rhs.equals_string(lhs); } #endif diff --git a/include/nlohmann/detail/meta/type_traits.hpp b/include/nlohmann/detail/meta/type_traits.hpp index a08ad1500..397bb6287 100644 --- a/include/nlohmann/detail/meta/type_traits.hpp +++ b/include/nlohmann/detail/meta/type_traits.hpp @@ -9,9 +9,11 @@ #pragma once #include // numeric_limits +#include // map, multimap #include // char_traits #include // tuple #include // false_type, is_constructible, is_integral, is_same, true_type +#include // unordered_map, unordered_multimap #include // declval #include // vector #if defined(__cpp_lib_byte) && __cpp_lib_byte >= 201603L @@ -683,6 +685,22 @@ struct is_compatible_integer_type : is_compatible_integer_type_impl {}; +// whether to_json() stores an integer of type CompatibleNumberIntegerType as +// number_integer +template +struct is_compatible_number_integer_type + : is_compatible_integer_type {}; + +// if number_unsigned_t is the same type as number_integer_t, unsigned integers +// are stored as number_integer as well (#2573) +template +struct is_compatible_number_integer_type, + std::is_integral, + std::is_unsigned, + negation>>::value>> + : std::is_constructible {}; + template struct is_compatible_type_impl: std::false_type {}; @@ -897,6 +915,31 @@ struct has_capacity : std::integral_constant struct is_ordered_map : has_capacity {}; +// whether T is a specialization of nlohmann::ordered_map, which only ever +// appends members, so they can only move when its capacity changes +template +struct is_nlohmann_ordered_map : std::false_type {}; + +template +struct is_nlohmann_ordered_map> : std::true_type {}; + +// whether adding or erasing elements never moves the other elements of the +// container T; only node-based standard containers are known to guarantee this +template +struct has_stable_elements : std::false_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + // to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > T conditional_static_cast(U value) @@ -1020,6 +1063,25 @@ constexpr bool value_in_range_of(T val) template using bool_constant = std::integral_constant; +// Stands in for number_unsigned_t in overload sets if it is the same type as +// number_integer_t, so that the unsigned overloads do not collide with the +// integer ones. Never constructed. Such a basic_json stores every integer as +// number_integer (#2573). +struct same_as_number_integer_type {}; + +// NumberUnsignedType, or same_as_number_integer_type if it is the same type as +// NumberIntegerType; basic_json uses this one, as it cannot name its own member +// types through an incomplete basic_json (MSVC reports a recursive alias, and +// old GCC versions crash) +template +using number_unsigned_overload_type = typename std::conditional::value, + same_as_number_integer_type, NumberUnsignedType>::type; + +// number_unsigned_t of BasicJsonType, or same_as_number_integer_type if it is +// the same type as number_integer_t +template +using number_unsigned_overload_t = number_unsigned_overload_type; + /////////////////////////////////////////////////////////////////////////////// // is_c_string /////////////////////////////////////////////////////////////////////////////// diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 2a18ea600..3a9abbc08 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -221,7 +221,7 @@ class binary_writer case value_t::number_unsigned: { - write_cbor_head(0x00, j.m_data.m_value.number_unsigned); + write_cbor_head(0x00, conditional_static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -1601,7 +1601,7 @@ class binary_writer return calc_bson_integer_size(j.m_data.m_value.number_integer); case value_t::number_unsigned: - return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); + return calc_bson_unsigned_size(conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return calc_bson_string_size(*j.m_data.m_value.string, j); @@ -1643,7 +1643,7 @@ class binary_writer return write_bson_integer(name, j.m_data.m_value.number_integer); case value_t::number_unsigned: - return write_bson_unsigned(name, j.m_data.m_value.number_unsigned); + return write_bson_unsigned(name, conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return write_bson_string(name, *j.m_data.m_value.string); diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 9a8203695..3621a78d9 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -49,6 +49,12 @@ class serializer using number_float_t = typename BasicJsonType::number_float_t; using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type of the absolute value of an integer: number_unsigned_t, or its + /// unsigned counterpart if it is signed (it may be number_integer_t, #2573) + using number_magnitude_t = typename std::conditional < + std::is_integral::value && std::is_signed::value, + std::make_unsigned, + std::enable_if>::type::type; using binary_char_t = typename BasicJsonType::binary_t::value_type; public: @@ -772,6 +778,33 @@ class serializer break; } } + + // A byte below 0x60 is a complete code point and needs no + // UTF-8 decoding. The scanner stops only at a control + // character, a quote, or a backslash in that range, so look up + // its escape sequence and write all 8 bytes of the table entry + // straight into the write buffer; only the length stored in the + // last byte of the entry is kept. + if (data[i] < 0x60) + { + // emit any bytes still pending in string_buffer first to + // preserve output order + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + const auto& escape = ascii_escapes()[data[i]]; + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size()); + write_buffer_pos += static_cast(escape[7]); + bytes_after_last_accept = 0; + undumped_chars = 0; + continue; + } } const auto byte = static_cast(s[i]); @@ -780,92 +813,39 @@ class serializer { case UTF8_ACCEPT: // decode found a new code point { - switch (codepoint) + // the code point is at least 0x60 (smaller ones are handled + // above) and is not printable ASCII (which the scanner + // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII + // character + if (EnsureAscii) { - case 0x08: // backspace + JSON_ASSERT(codepoint >= 0x7F); + if (error_handler == error_handler_t::keep) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; + // this character was buffered as raw bytes below + // in case it turned out to be part of an + // ill-formed sequence (which is kept as is); now + // that it decoded to a well-formed code point, + // undo that and \u-escape it like any other + // character instead + bytes = bytes_after_last_accept; } - - case 0x09: // horizontal tab + if (codepoint <= 0xFFFF) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; + write_u_escape(bytes, static_cast(codepoint)); } - - case 0x0A: // newline + else { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // EnsureAscii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) - { - if (EnsureAscii && error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes - // below in case it turned out to be part of - // an ill-formed sequence (which is kept as - // is); now that it decoded to a well-formed - // code point, undo that and \u-escape it - // like any other character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; + write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); + write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); } } + else + { + // copy byte to buffer (all previous bytes of the + // sequence have been copied in the default case below) + string_buffer[bytes++] = s[i]; + } // write buffer and reset index; there must be 13 bytes // left, as this is the maximal number of bytes to be @@ -1227,7 +1207,7 @@ class serializer @param[in] x unsigned integer number to count its digits @return number of decimal digits */ - unsigned int count_digits(number_unsigned_t x) noexcept + unsigned int count_digits(number_magnitude_t x) noexcept { unsigned int n_digits = 1; for (;;) @@ -1253,6 +1233,121 @@ class serializer } } + /*! + @brief the JSON form of each character below 0x60 + + Entry c holds the escape sequence of character c - the short form for + \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters - + or c itself if it needs no escaping, padded with zeros to 7 bytes. The last + byte is the length of the sequence. The 8 bytes of an entry are copied at + once, so writing an escape needs neither a branch nor a byte-wise loop. + */ + static const std::array, 0x60>& ascii_escapes() noexcept + { + static const std::array, 0x60> escapes = + { + { + {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00 + {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01 + {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02 + {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03 + {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04 + {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05 + {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06 + {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07 + {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08 + {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09 + {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A + {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B + {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C + {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D + {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E + {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F + {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10 + {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11 + {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12 + {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13 + {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14 + {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15 + {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16 + {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17 + {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18 + {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19 + {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A + {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B + {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C + {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D + {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E + {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F + {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20 + {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21 + {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22 + {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23 + {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24 + {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25 + {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26 + {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27 + {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28 + {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29 + {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A + {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B + {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C + {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D + {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E + {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F + {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30 + {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31 + {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32 + {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33 + {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34 + {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35 + {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36 + {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37 + {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38 + {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39 + {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A + {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B + {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C + {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D + {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E + {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F + {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40 + {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41 + {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42 + {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43 + {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44 + {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45 + {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46 + {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47 + {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48 + {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49 + {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A + {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B + {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C + {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D + {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E + {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F + {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50 + {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51 + {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52 + {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53 + {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54 + {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55 + {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56 + {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57 + {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58 + {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59 + {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A + {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B + {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C + {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D + {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E + {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F + } + }; + return escapes; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * @@ -1410,7 +1505,7 @@ class serializer } auto* buffer_ptr = write_buffer.data() + write_buffer_pos; - number_unsigned_t abs_value; + number_magnitude_t abs_value; // one byte for the minus sign unsigned int n_chars = 0; @@ -1423,7 +1518,7 @@ class serializer } else { - abs_value = static_cast(x); + abs_value = static_cast(x); } // up to 16 digits: eight at a time (as the digits of floats), written @@ -1591,17 +1686,6 @@ class serializer } } - /* - * Overload to make the compiler happy while it is instantiating - * dump_integer for number_unsigned_t. - * Must never be called. - */ - number_unsigned_t remove_sign(number_unsigned_t x) - { - JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE - return x; // LCOV_EXCL_LINE - } - /* * Helper function for dump_integer * @@ -1611,10 +1695,10 @@ class serializer * absolute values of INT_MIN and INT_MAX are usually not the same. See * #1708 for details. */ - number_unsigned_t remove_sign(number_integer_t x) noexcept + number_magnitude_t remove_sign(number_integer_t x) noexcept { JSON_ASSERT(x < 0); - return static_cast(-(x + 1)) + 1; + return static_cast(-(x + 1)) + 1; } private: diff --git a/include/nlohmann/detail/string_escape.hpp b/include/nlohmann/detail/string_escape.hpp index a6489e054..b3308b5c3 100644 --- a/include/nlohmann/detail/string_escape.hpp +++ b/include/nlohmann/detail/string_escape.hpp @@ -9,6 +9,7 @@ #pragma once #include // size_t +#include // move #include @@ -24,39 +25,46 @@ namespace detail * Note the order of escaping "~" to "~0" and "/" to "~1" is important. * * The string is rebuilt in a single pass, appending whole runs between the - * characters that need escaping. Scanning with find_first_of() keeps the - * common case -- nothing to escape -- as fast as a single search, while - * repeated replace() calls would move the tail of the string once per - * escaped character. + * characters that need escaping, while repeated replace() calls would move + * the tail of the string once per escaped character. @ref escape_into appends + * the escaped string to an existing one. */ +template +inline void escape_into(StringType& out, const StringType& s) +{ + const std::size_t size = s.size(); + std::size_t run = 0; + for (std::size_t i = 0; i < size; ++i) + { + const char c = s[i]; + if (c == '~' || c == '/') + { + out.append(s.data() + run, i - run); + out.append(c == '~' ? "~0" : "~1", 2); + run = i + 1; + } + } + out.append(s.data() + run, size - run); +} + template inline StringType escape(const StringType& s) { - auto next_special = [&s](std::size_t from) + // a single scan for both special characters; nothing to escape is the common case + const std::size_t size = s.size(); + std::size_t i = 0; + while (i < size && s[i] != '~' && s[i] != '/') { - const auto tilde = s.find_first_of('~', from); - const auto slash = s.find_first_of('/', from); - return tilde < slash ? tilde : slash; // npos is the largest value - }; - - auto pos = next_special(0); - if (pos == StringType::npos) + ++i; + } + if (i == size) { return s; } StringType result; - result.reserve(s.size() + 2); - - std::size_t run = 0; - while (pos != StringType::npos) - { - result.append(s.data() + run, pos - run); - result.append(s[pos] == '~' ? "~0" : "~1", 2); - run = pos + 1; - pos = next_special(run); - } - result.append(s.data() + run, s.size() - run); + result.reserve(size + 2); + escape_into(result, s); return result; } @@ -101,7 +109,7 @@ inline void unescape(StringType& s) pos = s.find_first_of('~', run); } result.append(s.data() + run, s.size() - run); - s = result; + s = std::move(result); } } // namespace detail diff --git a/include/nlohmann/detail/value_t.hpp b/include/nlohmann/detail/value_t.hpp index 9fc217bcc..cda0facd2 100644 --- a/include/nlohmann/detail/value_t.hpp +++ b/include/nlohmann/detail/value_t.hpp @@ -117,6 +117,34 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept } #endif +/*! +@brief whether a value of the given type owns heap memory + +Strings, binaries, objects, and arrays are stored as pointers to allocated +values; all other types are stored directly in the union. Every enumerator is +listed so that -Wswitch-enum flags this function when a new type is added. +*/ +inline bool owns_memory(const value_t t) noexcept +{ + switch (t) + { + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::binary: + return true; + + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + return false; + } +} + /*! @brief compare an integer with a floating point number without precision loss diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 3bad024f1..2ca6b2d3a 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -500,8 +500,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec "NumberUnsignedType must be at least as wide as NumberIntegerType, " "because it has to hold the absolute value of every NumberIntegerType value"); + // NumberUnsignedType may be the same type as NumberIntegerType (#2573), but + // a repeated NumberFloatType would give overloads that collide, so it gets + // a readable diagnostic instead. + static_assert(!std::is_same::value && !std::is_same::value, + "NumberFloatType must be a different type than NumberIntegerType and NumberUnsignedType"); + private: + /// the kind a value of kind @a t is stored as: if number_unsigned_t is the + /// same type as number_integer_t, every integer is stored as number_integer + /// (#2573); every other kind is stored as itself + static constexpr value_t stored_kind(const value_t t) noexcept + { + return (std::is_same::value && t == value_t::number_unsigned) ? value_t::number_integer : t; + } + /// helper for exception-safe object creation template JSON_HEDLEY_RETURNS_NON_NULL @@ -575,8 +589,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec json_value(boolean_t v) noexcept : boolean(v) {} /// constructor for numbers (integer) json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (unsigned); takes a placeholder if number_unsigned_t is number_integer_t + json_value(detail::number_unsigned_overload_type v) noexcept : number_unsigned(v) {} /// constructor for numbers (floating-point) json_value(number_float_t v) noexcept : number_float(v) {} /// constructor for empty values of a given type @@ -585,26 +599,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec switch (t) { case value_t::object: - { - object = create(); - break; - } - case value_t::array: - { - array = create(); - break; - } - case value_t::string: - { - string = create(""); - break; - } - case value_t::binary: { - binary = create(); + // allocating: kept out of line, so the scalar cases below inline + init_owned(t); break; } @@ -644,13 +644,41 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec object = nullptr; // silence warning, see #821 if (JSON_HEDLEY_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(detail::exception_id::internal_error, "961c151d2e87f2686a955a9be24d316f1362bf21 3.12.0", nullptr)); // LCOV_EXCL_LINE + detail::throw_literal(detail::exception_id::internal_error, "961c151d2e87f2686a955a9be24d316f1362bf21 3.12.0", nullptr); // LCOV_EXCL_LINE } break; } } } + JSON_HEDLEY_NEVER_INLINE + void init_owned(value_t t) + { + switch (t) + { + case value_t::object: + object = create(); + break; + case value_t::array: + array = create(); + break; + case value_t::string: + string = create(""); + break; + case value_t::binary: + binary = create(); + break; + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + break; + } + } + /// constructor for strings json_value(const string_t& value) : string(create(value)) {} @@ -720,6 +748,34 @@ private: } } + // true if no child of the non-empty array/object v has children of + // its own, i.e. v can be freed in one go (its destructor then + // destroys scalars and empty containers only, so it recurses at most + // one level deep); stops at the first structured child it meets + static bool all_children_leaves(const basic_json& v) noexcept + { + if (v.m_data.m_type == value_t::array) + { + const auto& a = *v.m_data.m_value.array; + for (auto it = a.rbegin(); it != a.rend(); ++it) + { + if (!has_no_children(*it)) + { + return false; + } + } + return true; + } + for (const auto& el : *v.m_data.m_value.object) + { + if (!has_no_children(el.second)) + { + return false; + } + } + return true; + } + // The walk in destroy_container() may take the children of a // container in any order, as long as it picks the same child again // while that container is not modified in between. Arrays and @@ -775,15 +831,16 @@ private: } } - // deallocates the (already empty) array/object held by v; this is - // the same allocator-based free the old recursive implementation - // used, just factored out so every level of the walk in destroy() - // can share it + // deallocates the array/object held by v, which is either empty or + // holds only scalars and empty containers (so its destructor + // recurses at most one level deep); this is the same allocator-based + // free the old recursive implementation used, just factored out so + // every level of the walk in destroy() can share it static void free_container(basic_json& v) noexcept { + JSON_ASSERT(has_no_children(v) || all_children_leaves(v)); if (v.m_data.m_type == value_t::array) { - JSON_ASSERT(v.m_data.m_value.array->empty()); AllocatorType alloc; std::allocator_traits::destroy(alloc, v.m_data.m_value.array); std::allocator_traits::deallocate(alloc, v.m_data.m_value.array, 1); @@ -791,7 +848,6 @@ private: else { JSON_ASSERT(v.m_data.m_type == value_t::object); - JSON_ASSERT(v.m_data.m_value.object->empty()); AllocatorType alloc; std::allocator_traits::destroy(alloc, v.m_data.m_value.object); std::allocator_traits::deallocate(alloc, v.m_data.m_value.object, 1); @@ -836,7 +892,58 @@ public: return; } - // Destroy the tree without recursing per nesting level and + // This json_value is not itself a basic_json, so the top-level + // container is first moved into a local stand-in ("cur"); a + // default-constructed basic_json has a null pointer in its + // m_value (see data::m_value's initializer), so swapping it + // with *this leaves this union's own pointer null, and it is + // never looked at or freed a second time. + basic_json cur; + cur.m_data.m_type = t; + using std::swap; + swap(cur.m_data.m_value, *this); + + destroy_bounded(cur, 0); + } + + // Destroy the array/object v, whose children are destroyed first. + // The first recursion_depth_limit() levels are destroyed by plain + // recursion, which is the fastest way and cannot throw; anything + // nested deeper is handed to destroy_walk(), which uses no call stack. + static void destroy_bounded(basic_json& v, std::size_t depth) noexcept + { + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + destroy_walk(v); + return; + } + + if (v.m_data.m_type == value_t::array) + { + for (auto& child : *v.m_data.m_value.array) + { + if (child.m_data.m_type == value_t::array || child.m_data.m_type == value_t::object) + { + destroy_bounded(child, depth + 1); + } + } + } + else + { + for (auto& child : *v.m_data.m_value.object) + { + if (child.second.m_data.m_type == value_t::array || child.second.m_data.m_type == value_t::object) + { + destroy_bounded(child.second, depth + 1); + } + } + } + + // only leaves and already destroyed (null) children remain + free_container(v); + } + + // Destroy the tree below top without recursing per nesting level and // without any heap allocation: a heap-allocated flattening // stack (the previous implementation) can itself throw // bad_alloc, which would escape this noexcept destructor and @@ -860,16 +967,24 @@ public: // pointer in its m_value (see data::m_value's initializer), // so swapping it with *this leaves this union's own pointer // null, and it is never looked at or freed a second time. + static void destroy_walk(basic_json& top) noexcept + { basic_json cur; - cur.m_data.m_type = t; - using std::swap; - swap(cur.m_data.m_value, *this); + take(cur, top); basic_json prev; // value_t::null: no parent + // true right after cur was entered: its children have not been + // looked at yet. If none of them has children of its own, cur is + // freed in one go instead of child by child; this is tried once + // per container, so the walk stays linear. + bool fresh = true; + while (true) { - if (has_no_children(cur)) + const bool fresh_now = fresh; + fresh = false; + if (has_no_children(cur) || (fresh_now && all_children_leaves(cur))) { if (prev.m_data.m_type == value_t::null) { @@ -908,10 +1023,22 @@ public: take(cur_child_ref, prev); take(prev, cur); take(cur, tmp); + fresh = true; } } + // keep the check whether t owns memory inline so destroying a scalar + // is a compare and a branch, and leave the actual work to an + // out-of-line function void destroy(value_t t) + { + if (JSON_HEDLEY_UNLIKELY(detail::owns_memory(t))) + { + destroy_owned(t); + } + } + + JSON_HEDLEY_NEVER_INLINE void destroy_owned(value_t t) { switch (t) { @@ -1079,11 +1206,16 @@ public: return next; } - /// @brief the capacity of the stored array, or unknown_size() - /// Only JSON_DIAGNOSTICS uses the value, to detect a reallocation that - /// would invalidate the parent pointers. Array types that do not have a - /// capacity() member function report unknown_size(), which is treated as - /// "the elements may have moved". + /// @brief the capacity of the stored array or object, or unknown_size() + /// Only JSON_DIAGNOSTICS uses the value, to detect an insertion that may + /// have moved the other elements and so invalidated their parent pointers. + /// unknown_size() means "the elements may have moved". Arrays only grow at + /// the end, so their capacity tells whether they reallocated; arrays + /// without a capacity() member report unknown_size(). For objects, only + /// types known to keep their elements in place (std::map and the other + /// node-based standard containers) report a fixed 0, and only + /// nlohmann::ordered_map, which appends members, reports its capacity; any + /// other object type (a sorted flat map, say) reports unknown_size(). #if JSON_DIAGNOSTICS template < typename A = array_t, detail::enable_if_t < detail::has_capacity::value, int > = 0 > std::size_t array_capacity() const noexcept @@ -1096,57 +1228,63 @@ public: { return detail::unknown_size(); } + + template < typename O = object_t, detail::enable_if_t < detail::has_stable_elements::value, int > = 0 > + static constexpr std::size_t object_capacity() noexcept + { + return 0; + } + + template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && detail::is_nlohmann_ordered_map::value, int > = 0 > + std::size_t object_capacity() const noexcept + { + return m_data.m_value.object->capacity(); + } + + template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && !detail::is_nlohmann_ordered_map::value, int > = 0 > + static constexpr std::size_t object_capacity() noexcept + { + return detail::unknown_size(); + } + + std::size_t container_capacity() const noexcept + { + return is_array() ? array_capacity() : object_capacity(); + } #else - static constexpr std::size_t array_capacity() noexcept + static constexpr std::size_t container_capacity() noexcept { return detail::unknown_size(); } #endif - /// @brief set the parent of a value that has just been added to an array - /// @param j the added value - /// @param old_capacity the value @ref array_capacity() returned before the + /// @brief set the parent of a value that has just been added to an array or object + /// @param j the added (or looked-up) value + /// @param old_capacity the value @ref container_capacity() returned before the /// insertion - reference set_parent_after_array_insert(reference j, std::size_t old_capacity) + reference set_parent_after_insert(reference j, std::size_t old_capacity) { #if JSON_DIAGNOSTICS // see https://github.com/nlohmann/json/issues/2838 - JSON_ASSERT(type() == value_t::array); if (JSON_HEDLEY_UNLIKELY(old_capacity == detail::unknown_size() - || array_capacity() != old_capacity)) + || container_capacity() != old_capacity)) { - // the capacity has changed, or the array type does not let us tell: - // the elements may have moved, so update all parents + // the capacity has changed, or the container type does not let us + // tell: the elements may have moved, so update all parents set_parents(); return j; } #else static_cast(old_capacity); #endif - return set_parent(j); + return set_parent_at(j); } - reference set_parent(reference j) + /// @brief set the parent of an element accessed via at(); never moves elements + reference set_parent_at(reference j) noexcept { #if JSON_DIAGNOSTICS - // ordered_json uses a vector internally, so pointers could have - // been invalidated; see https://github.com/nlohmann/json/issues/2962 -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - set_parents(); - return j; - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif - j.m_parent = this; -#else - static_cast(j); #endif return j; } @@ -2136,9 +2274,11 @@ public: /// @brief restore the parent pointers after erasing from an object - /// ordered_json keeps its members in a vector, and erasing a member - /// re-constructs every member after it in place, which resets their - /// parent pointers + /// Unless the object type is known to keep its elements in place (see + /// detail::has_stable_elements), erasing a member may move the members + /// after it; ordered_json, for instance, keeps its members in a vector + /// and re-constructs every member after the erased one in place, which + /// resets their parent pointers void set_parents_after_object_erase() { #if JSON_DIAGNOSTICS @@ -2146,7 +2286,7 @@ public: #pragma warning(push ) #pragma warning(disable : 4127) // ignore warning to replace if with if constexpr #endif - if (detail::is_ordered_map::value) + if (!detail::has_stable_elements::value) { set_parents(); } @@ -2181,7 +2321,7 @@ public: /// @brief create an empty value with a given type /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(const value_t v) - : m_data(v) + : m_data(stored_kind(v)) { assert_invariant(); } @@ -2262,7 +2402,7 @@ public: { convert_leaf(val); } - JSON_ASSERT(m_data.m_type == val.type()); + JSON_ASSERT(m_data.m_type == stored_kind(val.type())); set_parents(); assert_invariant(); @@ -2310,7 +2450,7 @@ public: // if an object is wanted but impossible, throw an exception if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object)) { - JSON_THROW(type_error::create(detail::exception_id::object_from_non_pairs, "cannot create object from initializer list", nullptr)); + detail::throw_literal(detail::exception_id::object_from_non_pairs, "cannot create object from initializer list", nullptr); } } @@ -2430,7 +2570,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterators_incompatible, "iterators are not compatible", nullptr)); + detail::throw_literal(detail::exception_id::iterators_incompatible, "iterators are not compatible", nullptr); } // copy type from the first iterator @@ -2449,7 +2589,7 @@ public: if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() || !last.m_it.primitive_iterator.is_end())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_out_of_range, "iterators out of range", first.m_object)); + detail::throw_literal(detail::exception_id::iterator_range_out_of_range, "iterators out of range", first.m_object); } break; } @@ -2517,7 +2657,7 @@ public: case value_t::null: case value_t::discarded: default: - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", first.m_object->type_name()), first.m_object)); + throw_iterator_range_of_null(first.m_object); } set_parents(); @@ -2546,18 +2686,16 @@ public: // check of passed value is valid other.assert_invariant(); - if (m_data.m_type == value_t::object || m_data.m_type == value_t::array) + if (JSON_HEDLEY_LIKELY(!detail::owns_memory(m_data.m_type))) { - // copying the container directly would call this constructor again - // for every element, once per nesting level - copy_structured(other); + // null, boolean, numbers, discarded: plain copy of the union + m_data.m_value = other.m_data.m_value; } else { - copy_leaf_value(other, *this); + copy_owning(other); } - set_parents(); assert_invariant(); } @@ -2787,6 +2925,22 @@ public: // value access // ////////////////// + /// out-of-line part of the copy constructor for owning types + JSON_HEDLEY_NEVER_INLINE + void copy_owning(const basic_json& other) + { + if (m_data.m_type == value_t::object || m_data.m_type == value_t::array) + { + // copying the container directly would call this constructor again + // for every element, once per nesting level + copy_structured(other); + } + else + { + copy_leaf_value(other, *this); + } + } + /// get a pointer to the value (object) object_t* get_impl_ptr(object_t* /*unused*/) noexcept { @@ -2847,14 +3001,16 @@ public: return m_data.m_type == value_t::number_integer ? &m_data.m_value.number_integer : nullptr; } - /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + /// get a pointer to the value (unsigned number); takes a placeholder if + /// number_unsigned_t is number_integer_t (the integer overload handles it then) + number_unsigned_t* get_impl_ptr(detail::number_unsigned_overload_type* /*unused*/) noexcept { return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } - /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + /// get a pointer to the value (unsigned number); takes a placeholder if + /// number_unsigned_t is number_integer_t (the integer overload handles it then) + constexpr const number_unsigned_t* get_impl_ptr(const detail::number_unsigned_overload_type* /*unused*/) const noexcept { return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } @@ -2883,6 +3039,40 @@ public: return is_binary() ? m_data.m_value.binary : nullptr; } + /// out-of-line throw helpers (message construction kept out of the hot paths) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_iterator_range_of_null(const basic_json* obj) + { + JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", obj->type_name()), obj)); + } + + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_incompatible_reference_type(const basic_json& obj) + { + JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); + } + + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_array_index_exceeds_size_type(size_type idx) + { + JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); + } + + /// @throw out_of_range.401 + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_array_index_out_of_range(size_type idx, const basic_json& j) + { + JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + } + + /// @throw out_of_range.403 + template + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_key_not_found(const KeyType& key, const basic_json& j) + { + JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key, "' not found"), &j)); + } + /*! @brief helper function to implement get_ref() @@ -2905,7 +3095,7 @@ public: return *ptr; } - JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); + throw_incompatible_reference_type(obj); } public: @@ -3353,7 +3543,7 @@ public: // std::map or ordered_map) never moves from its argument, so key is still // valid here regardless of whether KeyType was deduced as an rvalue reference // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved) - JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key_for_message(key), "' not found"), &j)); + throw_key_not_found(key_for_message(key), j); } return it->second; } @@ -3390,7 +3580,7 @@ public: if (JSON_HEDLEY_UNLIKELY(idx >= j.m_data.m_value.array->size())) { - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + throw_array_index_out_of_range(idx, j); } return (*j.m_data.m_value.array)[idx]; @@ -3445,7 +3635,7 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(size_type idx) { - return set_parent(array_at(*this, idx)); + return set_parent_at(array_at(*this, idx)); } /// @brief access specified array element with bounds checking @@ -3459,7 +3649,7 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(const typename object_t::key_type& key) { - return set_parent(object_at(*this, key)); + return set_parent_at(object_at(*this, key)); } /// @brief access specified object element with bounds checking @@ -3468,7 +3658,7 @@ public: detail::is_usable_as_basic_json_key_type::value, int> = 0> reference at(KeyType && key) { - return set_parent(object_at(*this, std::forward(key))); + return set_parent_at(object_at(*this, std::forward(key))); } /// @brief access specified object element with bounds checking @@ -3508,18 +3698,18 @@ public: // resize() rejects other indices that are too large to represent if (JSON_HEDLEY_UNLIKELY(idx == (std::numeric_limits::max)())) { - JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); + throw_array_index_exceeds_size_type(idx); } #if JSON_DIAGNOSTICS // remember array size & capacity before resizing const auto old_size = m_data.m_value.array->size(); - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); #endif m_data.m_value.array->resize(idx + 1); #if JSON_DIAGNOSTICS if (JSON_HEDLEY_UNLIKELY(old_capacity == detail::unknown_size() - || array_capacity() != old_capacity)) + || container_capacity() != old_capacity)) { // capacity has changed: update all parents set_parents(); @@ -3566,8 +3756,9 @@ public: // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { + const auto old_capacity = container_capacity(); auto result = m_data.m_value.object->emplace(std::move(key), nullptr); - return set_parent(result.first->second); + return set_parent_after_insert(result.first->second, old_capacity); } throw_subscript_wrong_type("string"); @@ -3617,8 +3808,9 @@ public: // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { + const auto old_capacity = container_capacity(); auto result = m_data.m_value.object->emplace(lookup_key(std::forward(key)), nullptr); - return set_parent(result.first->second); + return set_parent_after_insert(result.first->second, old_capacity); } throw_subscript_wrong_type("string"); @@ -3844,7 +4036,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(this != pos.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } IteratorType result = end(); @@ -3860,7 +4052,7 @@ public: { if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_out_of_range, "iterator out of range", this)); + detail::throw_literal(detail::exception_id::iterator_out_of_range, "iterator out of range", this); } m_data.m_value.destroy(m_data.m_type); @@ -3902,7 +4094,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_from_other_value, "iterators do not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_range_from_other_value, "iterators do not fit current value", this); } IteratorType result = end(); @@ -3919,7 +4111,7 @@ public: if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() || !last.m_it.primitive_iterator.is_end())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_out_of_range, "iterators out of range", this)); + detail::throw_literal(detail::exception_id::iterator_range_out_of_range, "iterators out of range", this); } m_data.m_value.destroy(m_data.m_type); @@ -4018,7 +4210,7 @@ public: { if (JSON_HEDLEY_UNLIKELY(idx >= size())) { - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); + throw_array_index_out_of_range(idx, *this); } m_data.m_value.array->erase(m_data.m_value.array->begin() + static_cast(idx)); @@ -4524,9 +4716,9 @@ public: } // add the element to the array (move semantics) - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->push_back(std::move(val)); - set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + set_parent_after_insert(m_data.m_value.array->back(), old_capacity); // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor } @@ -4555,9 +4747,9 @@ public: } // add the element to the array - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->push_back(val); - set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + set_parent_after_insert(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an array @@ -4585,8 +4777,9 @@ public: } // add the element to the object + const auto old_capacity = container_capacity(); auto res = m_data.m_value.object->insert(val); - set_parent(res.first->second); + set_parent_after_insert(res.first->second, old_capacity); } /// @brief add an object to an object @@ -4639,9 +4832,9 @@ public: } // add the element to the array (perfect forwarding) - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->emplace_back(std::forward(args)...); - return set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + return set_parent_after_insert(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an object if key does not exist @@ -4662,8 +4855,9 @@ public: } // add the element to the array (perfect forwarding) + const auto old_capacity = container_capacity(); auto res = m_data.m_value.object->emplace(std::forward(args)...); - set_parent(res.first->second); + set_parent_after_insert(res.first->second, old_capacity); // create a result iterator and set iterator to the result of emplace auto it = begin(); @@ -4704,7 +4898,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // insert to array and return iterator @@ -4724,7 +4918,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // moving into a local first keeps this safe even if val aliases @@ -4746,7 +4940,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // insert to array and return iterator @@ -4769,24 +4963,24 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } if (JSON_HEDLEY_UNLIKELY(first.m_object == this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_into_itself, "passed iterators may not belong to container", this)); + detail::throw_literal(detail::exception_id::insert_range_into_itself, "passed iterators may not belong to container", this); } // passed iterators must belong to arrays if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_array())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterators first and last must point to arrays", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to arrays", this); } // insert to array and return iterator @@ -4806,7 +5000,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // copy the values first: ilist may refer to elements of this array @@ -4834,13 +5028,13 @@ public: // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } // passed iterators must belong to objects if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this); } m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); @@ -4875,7 +5069,7 @@ public: // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } // passed iterators must belong to objects @@ -4962,9 +5156,9 @@ public: continue; } } - // set_parent() also repairs the other members, which ordered_json - // relocates when adding a key makes its vector grow - set_parent(m_data.m_value.object->operator[](it.key()) = std::move(it.value())); + // operator[] repairs the other members if the object type moves + // them; assigning through it keeps this O(1) per key + operator[](it.key()) = std::move(it.value()); } } @@ -5014,9 +5208,8 @@ public: continue; } } - // set_parent() also repairs the other members, which ordered_json - // relocates when adding a key makes its vector grow - target->set_parent(target->m_data.m_value.object->operator[](first.key()) = std::move(first.value())); + // operator[] repairs the other members if the object type moves them + target->operator[](first.key()) = std::move(first.value()); ++first; } } @@ -5158,7 +5351,22 @@ public: /// @name lexicographical comparison operators /// @{ - // note parentheses around operands are necessary; see + // the same-type cases for non-owning scalars, shared by JSON_IMPLEMENT_OPERATOR + // and JSON_IMPLEMENT_SCALAR_FAST +#define JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + case value_t::null: \ + return (null_result); \ + case value_t::boolean: \ + return lhs.m_data.m_value.boolean op rhs.m_data.m_value.boolean; \ + case value_t::number_integer: \ + return lhs.m_data.m_value.number_integer op rhs.m_data.m_value.number_integer; \ + case value_t::number_unsigned: \ + return lhs.m_data.m_value.number_unsigned op rhs.m_data.m_value.number_unsigned; \ + case value_t::number_float: \ + return lhs.m_data.m_value.number_float op rhs.m_data.m_value.number_float; + + // note parentheses around the container operands are necessary, because + // GCC otherwise parses `array <` as std::array; see // https://github.com/nlohmann/json/issues/1530 // Mixed signed/unsigned integer comparisons check whether the signed value // is negative before casting. If it is, the comparison is performed with @@ -5194,24 +5402,11 @@ public: return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ } \ \ - case value_t::null: \ - return (null_result); \ - \ + JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + \ case value_t::string: \ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ - case value_t::boolean: \ - return (lhs.m_data.m_value.boolean) op (rhs.m_data.m_value.boolean); \ - \ - case value_t::number_integer: \ - return (lhs.m_data.m_value.number_integer) op (rhs.m_data.m_value.number_integer); \ - \ - case value_t::number_unsigned: \ - return (lhs.m_data.m_value.number_unsigned) op (rhs.m_data.m_value.number_unsigned); \ - \ - case value_t::number_float: \ - return (lhs.m_data.m_value.number_float) op (rhs.m_data.m_value.number_float); \ - \ case value_t::binary: \ return (*lhs.m_data.m_value.binary) op (*rhs.m_data.m_value.binary); \ \ @@ -5255,6 +5450,24 @@ public: \ return (default_result); + // inline fast path for two values of the same non-owning scalar type; anything + // else falls through to the out-of-line general comparison +#define JSON_IMPLEMENT_SCALAR_FAST(op, null_result) \ + if (lhs.m_data.m_type == rhs.m_data.m_type) \ + { \ + switch (lhs.m_data.m_type) \ + { \ + JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + case value_t::object: \ + case value_t::array: \ + case value_t::string: \ + case value_t::binary: \ + case value_t::discarded: \ + default: \ + break; \ + } \ + } + JSON_PRIVATE_UNLESS_TESTED: // returns true if: // - any operand is NaN and the other operand is of number type @@ -5282,6 +5495,173 @@ public: return compares_unordered(*this, rhs, inverse); } + /// @brief whether objects and arrays are the standard containers, whose + /// elements equal_bounded() may therefore compare itself + using standard_object_t = std::map>>; + using standard_array_t = std::vector>; + using equal_bounded_containers = detail::conjunction, std::is_same>; + + // out-of-line general comparisons; the operators below handle same-type + // scalars inline and fall through to these + JSON_HEDLEY_NEVER_INLINE + static bool equal_general(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + JSON_IMPLEMENT_OPERATOR( ==, true, false, false, + compare_iteratively(lhs, rhs, false) == compare_result::equal, true) +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + } + + JSON_HEDLEY_NEVER_INLINE + static bool less_general(const_reference lhs, const_reference rhs) noexcept + { + // default_result is used if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + compare_iteratively(lhs, rhs, true) == compare_result::less, false) + } + +#if JSON_HAS_THREE_WAY_COMPARISON + JSON_HEDLEY_NEVER_INLINE + static std::partial_ordering three_way_general(const_reference lhs, const_reference rhs) noexcept // *NOPAD* + { + // default_result is used if we cannot compare values. In that case, + // we compare types. + JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* + std::partial_ordering::equivalent, + std::partial_ordering::unordered, + lhs_type <=> rhs_type, // *NOPAD* + to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) + } +#endif + + /// @brief compare two leaves of the same type inside equal_structured() + /// @return 1 if they are equal, 0 if they are not, and -1 if the values are + /// of different types or are not handled here, which this leaves to + /// equal_structured() + /// @note Unlike JSON_IMPLEMENT_SCALAR_FAST, this also compares strings: + /// they are the most common leaves in a document, so comparing them + /// inline pays off in the container walk, while the operators keep + /// their inlined code at each call site small. Binary values are + /// rare and left to equal_general(). + JSON_HEDLEY_ALWAYS_INLINE + static int equal_leaf(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + if (lhs.m_data.m_type == rhs.m_data.m_type) + { + switch (lhs.m_data.m_type) + { + case value_t::null: + return 1; + case value_t::boolean: + return lhs.m_data.m_value.boolean == rhs.m_data.m_value.boolean; + case value_t::number_integer: + return lhs.m_data.m_value.number_integer == rhs.m_data.m_value.number_integer; + case value_t::number_unsigned: + return lhs.m_data.m_value.number_unsigned == rhs.m_data.m_value.number_unsigned; + case value_t::number_float: + return lhs.m_data.m_value.number_float == rhs.m_data.m_value.number_float; + case value_t::string: + return *lhs.m_data.m_value.string == *rhs.m_data.m_value.string; + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + } +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + return -1; + } + + /// @brief compare for equality, walking objects and arrays here instead of + /// in their containers so that the depth is a parameter and needs no + /// thread-local count; only used for the standard containers + static bool equal_bounded(const_reference lhs, const_reference rhs, std::size_t depth) noexcept + { + const int leaf = equal_leaf(lhs, rhs); + return leaf < 0 ? equal_structured(lhs, rhs, depth) : leaf != 0; + } + + static bool equal_structured(const_reference lhs, const_reference rhs, std::size_t depth) noexcept + { + const auto type = lhs.m_data.m_type; + if (type != rhs.m_data.m_type || (type != value_t::array && type != value_t::object)) + { + return equal_general(lhs, rhs); + } + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + return compare_iteratively(lhs, rhs, false) == compare_result::equal; + } + + if (type == value_t::array) + { + const auto& a = *lhs.m_data.m_value.array; + const auto& b = *rhs.m_data.m_value.array; + if (a.size() != b.size()) + { + return false; + } + auto it_b = b.cbegin(); + for (auto it_a = a.cbegin(); it_a != a.cend(); ++it_a, ++it_b) + { + const int leaf = equal_leaf(*it_a, *it_b); + if (leaf < 0 ? !equal_structured(*it_a, *it_b, depth + 1) : leaf == 0) + { + return false; + } + } + return true; + } + + const auto& a = *lhs.m_data.m_value.object; + const auto& b = *rhs.m_data.m_value.object; + if (a.size() != b.size()) + { + return false; + } + auto it_b = b.cbegin(); + for (auto it_a = a.cbegin(); it_a != a.cend(); ++it_a, ++it_b) + { + if (!(it_a->first == it_b->first)) + { + return false; + } + const int leaf = equal_leaf(it_a->second, it_b->second); + if (leaf < 0 ? !equal_structured(it_a->second, it_b->second, depth + 1) : leaf == 0) + { + return false; + } + } + return true; + } + + static bool equal_dispatch(const_reference lhs, const_reference rhs, std::true_type /*standard containers*/) noexcept + { + return equal_bounded(lhs, rhs, 0); + } + + static bool equal_dispatch(const_reference lhs, const_reference rhs, std::false_type /*other containers*/) noexcept + { + return equal_general(lhs, rhs); + } + public: #if JSON_HAS_THREE_WAY_COMPARISON /// @brief comparison: equal @@ -5293,11 +5673,11 @@ public: JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif const_reference lhs = *this; - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, - compare_iteratively(lhs, rhs, false) == compare_result::equal, true) + JSON_IMPLEMENT_SCALAR_FAST( ==, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP #endif + return equal_dispatch(lhs, rhs, equal_bounded_containers {}); } /// @brief comparison: equal @@ -5314,13 +5694,8 @@ public: std::partial_ordering operator<=>(const_reference rhs) const noexcept // *NOPAD* { const_reference lhs = *this; - // default_result is used if we cannot compare values. In that case, - // we compare types. - JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* - std::partial_ordering::equivalent, - std::partial_ordering::unordered, - lhs_type <=> rhs_type, // *NOPAD* - to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) + JSON_IMPLEMENT_SCALAR_FAST(<=>, std::partial_ordering::equivalent) // *NOPAD* + return three_way_general(lhs, rhs); } /// @brief comparison: 3-way @@ -5408,11 +5783,11 @@ public: JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, - compare_iteratively(lhs, rhs, false) == compare_result::equal, true) + JSON_IMPLEMENT_SCALAR_FAST( ==, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP #endif + return equal_dispatch(lhs, rhs, equal_bounded_containers {}); } // The friend comparisons with a scalar name the JSON type via decltype of @@ -5467,11 +5842,8 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ friend bool operator<(const_reference lhs, const_reference rhs) noexcept { - // default_result is used if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), - compare_iteratively(lhs, rhs, true) == compare_result::less, false) + JSON_IMPLEMENT_SCALAR_FAST( <, false) + return less_general(lhs, rhs); } /// @brief comparison: less than @@ -5582,6 +5954,8 @@ public: #endif #undef JSON_IMPLEMENT_OPERATOR +#undef JSON_IMPLEMENT_SCALAR_FAST +#undef JSON_IMPLEMENT_SCALAR_CASES /// @} @@ -6562,24 +6936,18 @@ public: }; // wrapper for "add" operation; add value at ptr - const auto operation_add = [&result](json_pointer & ptr, const basic_json & val) + const auto operation_add = [&result](json_pointer & ptr, basic_json val) { // adding to the root of the target document means replacing it if (ptr.empty()) { - result = val; + result = std::move(val); return; } - // make sure the top element of the pointer exists - json_pointer const top_pointer = ptr.top(); - if (top_pointer != ptr) - { - result.at(top_pointer); - } - - // get reference to the parent of the JSON pointer ptr - const auto last_path = ptr.back(); + // get reference to the parent of the JSON pointer ptr; a missing + // parent (or one of its ancestors) is reported by at() + auto last_path = std::move(ptr.reference_tokens.back()); ptr.pop_back(); // parent must exist when performing patch add per RFC6902 specs basic_json& parent = result.at(ptr); @@ -6590,7 +6958,7 @@ public: case value_t::object: { // use operator[] to add value - parent[last_path] = val; + parent[std::move(last_path)] = std::move(val); break; } @@ -6599,7 +6967,7 @@ public: if (last_path == "-") { // special case: append to back - parent.push_back(val); + parent.push_back(std::move(val)); } else { @@ -6607,11 +6975,11 @@ public: if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) { // avoid undefined behavior - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &parent)); + throw_array_index_out_of_range(idx, parent); } // default case: insert add offset - parent.insert(parent.begin() + static_cast(idx), val); + parent.insert(parent.begin() + static_cast(idx), std::move(val)); } break; } @@ -6626,7 +6994,7 @@ public: case value_t::binary: case value_t::discarded: default: - JSON_THROW(out_of_range::create(detail::exception_id::patch_add_parent_not_container, detail::concat("cannot add value: the JSON Patch 'add' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + detail::throw_patch_add_parent(parent); } }; @@ -6634,7 +7002,8 @@ public: const auto operation_remove = [this, & result](json_pointer & ptr) { // get reference to the parent of the JSON pointer ptr - const auto last_path = ptr.back(); + static_cast(ptr.back()); // throws out_of_range.405 if ptr is the root + const auto last_path = std::move(ptr.reference_tokens.back()); ptr.pop_back(); basic_json& parent = result.at(ptr); @@ -6649,7 +7018,7 @@ public: } else { - JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", last_path, "' not found"), this)); + throw_key_not_found(last_path, *this); } } else if (parent.is_array()) @@ -6661,7 +7030,7 @@ public: { // the parent of a "remove" target must be an object or array // (see #5396) - JSON_THROW(out_of_range::create(detail::exception_id::patch_remove_parent_not_container, detail::concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + detail::throw_patch_remove_parent(parent); } }; @@ -6692,7 +7061,7 @@ public: // type check: top level value must be an array if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array())) { - JSON_THROW(parse_error::create(detail::exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &json_patch)); + detail::throw_patch_not_an_array(json_patch); } // iterate and apply the operations @@ -6706,21 +7075,16 @@ public: // find value auto it = val.m_data.m_value.object->find(member); - // context-sensitive error message - const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\''); // NOLINT(bugprone-unused-local-non-trivial-variable) - // check if the desired value is present if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end())) { - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat(error_msg, " must have member '", member, "'"), &val)); + detail::throw_patch_missing_member(op, member, false, val); } // check if the result is of type string if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) { - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat(error_msg, " must have string member '", member, "'"), &val)); + detail::throw_patch_missing_member(op, member, true, val); } // no error: return value @@ -6730,12 +7094,12 @@ public: // type check: every element of the array must be an object if (JSON_HEDLEY_UNLIKELY(!val.is_object())) { - JSON_THROW(parse_error::create(detail::exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &val)); + detail::throw_patch_not_an_array(val); } // collect mandatory members - const auto op = get_value("op", "op", true).template get(); - const auto path = get_value(op, "path", true).template get(); + const string_t& op = *get_value("op", "op", true).m_data.m_value.string; + const string_t& path = *get_value(op, "path", true).m_data.m_value.string; json_pointer ptr(path); switch (get_op(op)) @@ -6761,38 +7125,37 @@ public: case patch_operations::move: { - const auto from_path = get_value("move", "from", true).template get(); + const string_t& from_path = *get_value("move", "from", true).m_data.m_value.string; json_pointer from_ptr(from_path); if (JSON_HEDLEY_UNLIKELY(is_proper_prefix(from_ptr, ptr))) { - JSON_THROW(out_of_range::create(detail::exception_id::patch_move_into_child, detail::concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); + detail::throw_patch_move_into_child(from_path, path, result); } // the "from" location must exist - use at() - basic_json const v = result.at(from_ptr); + // a root "from" is rejected by operation_remove below, which must leave result intact + basic_json v = from_ptr.empty() ? basic_json(result) : std::move(result.at(from_ptr)); // The move operation is functionally identical to a // "remove" operation on the "from" location, followed // immediately by an "add" operation at the target // location with the value that was just removed. operation_remove(from_ptr); - operation_add(ptr, v); + operation_add(ptr, std::move(v)); break; } case patch_operations::copy: { - const auto from_path = get_value("copy", "from", true).template get(); + const string_t& from_path = *get_value("copy", "from", true).m_data.m_value.string; const json_pointer from_ptr(from_path); - // the "from" location must exist - use at() - basic_json const v = result.at(from_ptr); - // The copy is functionally identical to an "add" // operation at the target location using the value - // specified in the "from" member. - operation_add(ptr, v); + // specified in the "from" member; the "from" location + // must exist - use at() + operation_add(ptr, result.at(from_ptr)); break; } @@ -6813,7 +7176,7 @@ public: // throw an exception if the test fails if (JSON_HEDLEY_UNLIKELY(!success)) { - JSON_THROW(other_error::create(detail::exception_id::patch_test_failed, detail::concat("unsuccessful: ", val.dump()), &val)); + detail::throw_patch_test_failed(val); } break; @@ -6824,7 +7187,7 @@ public: { // op must be "add", "remove", "replace", "move", "copy", or // "test" - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat("operation value '", op, "' is invalid"), &val)); + detail::throw_patch_invalid_operation(op, val); } } } @@ -6878,12 +7241,11 @@ public: std::size_t index = 0; /// objects: the next member of source to look at const_iterator member{}; // NOLINT(readability-redundant-member-init) - /// objects: the keys common to both, in source's order - std::vector common_keys{}; // NOLINT(readability-redundant-member-init) - /// objects: the next entry of common_keys - std::size_t next_common = 0; - /// objects: the "add" operations for keys only target has - basic_json added_ops{}; // NOLINT(readability-redundant-member-init) + /// objects: for each member of source (in order), its value in + /// target, or nullptr if target has no such key; `index` is the next + std::vector target_values{}; // NOLINT(readability-redundant-member-init) + /// objects: the members only target has, in target's order + std::vector added{}; // NOLINT(readability-redundant-member-init) }; // The operations of a diff are built by the functions below rather than @@ -6939,111 +7301,113 @@ public: } } + /// @brief appends the "add" operations for the members @a added to @a result + static void diff_added(basic_json& result, const std::vector& added, const string_t& path) + { + for (const auto& it : added) + { + diff_add(result, detail::concat(path, '/', detail::escape(it.key())), it.value()); + } + } + /*! @brief compare the keys of objects @a source and @a target If object_t does not keep its members in insertion order, or if the keys both objects have are in the same order in both, and the keys only - @a target has come after them, stores the keys common to both in - source's order in @a common_keys, stores the "add" operations for the keys - only @a target has in @a added_ops, and returns true: the caller then diffs - the objects member by member. Otherwise, appends operations that remove - every member of @a source and add every member of @a target to @a result, - and returns false. + @a target has come after them, stores in @a target_values, for each member + of source in order, its value in target (nullptr for a key target does not + have), stores the members only target has in @a added, and returns true: + the caller then diffs the objects member by member. Otherwise, appends + operations that remove every member of @a source and add every member of + @a target to @a result, and returns false. */ static bool diff_object_keys(basic_json& result, const basic_json& source, const basic_json& target, - const string_t& path, std::vector& common_keys, - basic_json& added_ops) + const string_t& path, std::vector& target_values, + std::vector& added) { - // first pass: record, for every source key, whether it is - // common to both objects (in source's iteration order) or - // was deleted (i.e., in source but not in target) -- this is - // a by-product of the target.find() call already needed to - // tell the two cases apart, so it adds no extra lookups. The - // "remove" ops themselves are emitted later, interleaved - // with the per-key diffs in the caller's fast path, to match - // source's original iteration order (as the original, - // pre-reordering-aware implementation did) instead of - // grouping all removes before all per-key diffs. - std::vector common_keys_source_order; + // first pass: one lookup per source key + target_values.reserve(source.size()); + std::size_t common = 0; for (auto it = source.cbegin(); it != source.cend(); ++it) { - if (target.find(it.key()) != target.end()) + const auto found = target.find(it.key()); + if (found != target.cend()) { - common_keys_source_order.push_back(it.key()); - } - } - - // second pass: find keys that were added (i.e., in target but - // not in source), and record the keys common to both, in - // target's iteration order -- again a by-product of the - // source.find() call already needed to detect added keys. At - // the same time, determine whether every added key comes - // after every common key in target's order (a precondition - // for the fast path, which only ever appends new keys - // at the very end). Both are only needed for an object_t that - // keeps its members in insertion order, such as the one - // backing `ordered_json`; for any other object_t, the fast - // path is always taken and they are not computed. - // The patch ops for keys that were added (i.e., in target but not - // in source) are built here so the fast path can reuse - // them without a second source.find() per target key. Only - // used by the fast path -- the slow (reordering) path - // rebuilds "add" ops for every key itself. - std::vector common_keys_target_order; - bool new_keys_form_suffix = true; - bool seen_new_key = false; - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - if (source.find(it.key()) == source.end()) - { - seen_new_key = true; - diff_add(added_ops, detail::concat(path, '/', detail::escape(it.key())), it.value()); + target_values.push_back(&found.value()); + ++common; } else { -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - common_keys_target_order.push_back(it.key()); - if (seen_new_key) - { - new_keys_form_suffix = false; - } - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif + target_values.push_back(nullptr); } } - // Only an object type that keeps its members in insertion - // order, such as nlohmann::ordered_map, can need reordering: - // patch() appends a new member at the end of such an object. - // Any other object type places its members itself - std::map - // in key order, a hash map in an order its operator== ignores - - // so a member-by-member diff always reproduces target there. + // second pass: find the keys only target has. An object type that + // keeps its members in insertion order, such as the one backing + // ordered_json, also needs the common keys to be in the same order + // in both objects, and the new keys to come after them, as patch() + // appends a new member at the end of such an object. Any other + // object type places its members itself, so a member-by-member diff + // always reproduces target there. #ifdef JSON_HEDLEY_MSVC_VERSION #pragma warning(push ) #pragma warning(disable : 4127) // ignore warning to replace if with if constexpr #endif - if (!detail::is_ordered_map::value - || (common_keys_source_order == common_keys_target_order && new_keys_form_suffix)) + bool fast_path = true; + if (detail::is_ordered_map::value) { - // fast path: order of common keys already matches (or the - // object_t's iteration order does not depend on - // insertion history), so a plain per-key diff is correct - // and minimal, as before - common_keys = std::move(common_keys_source_order); - return true; + // walk the common keys of source in lockstep with target: a key + // of target that is not the next one is either new or out of order + auto src = source.cbegin(); + std::size_t next = 0; + bool seen_new = false; + for (auto it = target.cbegin(); it != target.cend() && fast_path; ++it) + { + while (next < target_values.size() && target_values[next] == nullptr) + { + ++src; + ++next; + } + + if (next < target_values.size() && src.key() == it.key()) + { + // a common key after a new one: new keys are no suffix + fast_path = !seen_new; + ++src; + ++next; + } + else if (source.find(it.key()) == source.cend()) + { + seen_new = true; + added.push_back(it); + } + else + { + // a common key out of order + fast_path = false; + } + } + } + else if (target.size() != common) + { + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.cend()) + { + added.push_back(it); + } + } } #ifdef JSON_HEDLEY_MSVC_VERSION #pragma warning( pop ) #endif + if (fast_path) + { + return true; + } + // slow path: the common keys are in a different relative // order in source and target (only possible for a // reorderable object_t like ordered_map). Building a @@ -7093,6 +7457,14 @@ public: return; } + diff_unequal(result, source, target, path, depth); + } + + /// @brief @ref diff_recursively for values known to be unequal; callers + /// compare first so they build no path for equal ones + static void diff_unequal(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, const std::size_t depth) + { if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) { diff_iteratively(result, source, target, path); @@ -7115,7 +7487,10 @@ public: while (i < source.size() && i < target.size()) { // recursive call to compare array values at index i - diff_recursively(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); + if (!(source[i] == target[i])) + { + diff_unequal(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); + } ++i; } @@ -7127,25 +7502,24 @@ public: case value_t::object: { - std::vector common_keys; - basic_json added_ops(value_t::array); - if (diff_object_keys(result, source, target, path, common_keys, added_ops)) + std::vector target_values; + std::vector added; + if (diff_object_keys(result, source, target, path, target_values, added)) { - // fast path: common_keys is, by construction, the - // subsequence of source's keys that are common to both - // objects, in source's iteration order -- so it can be - // walked in lockstep with `source` using a cheap key - // comparison instead of another lookup. Deleted keys - // (those source keys not in common_keys) are interleaved - // here too, in source's original order, to match the + // fast path: target_values holds, for every member of + // source in order, its value in target -- so no further + // lookup is needed. Deleted keys are interleaved here + // too, in source's original order, to match the // historical (pre-reordering-aware) output order. - auto common_it = common_keys.cbegin(); - for (auto it = source.cbegin(); it != source.cend(); ++it) + std::size_t i = 0; + for (auto it = source.cbegin(); it != source.cend(); ++it, ++i) { - if (common_it != common_keys.cend() && it.key() == *common_it) + if (target_values[i] != nullptr) { - diff_recursively(result, it.value(), target[it.key()], detail::concat(path, '/', detail::escape(it.key())), depth + 1); - ++common_it; + if (!(it.value() == *target_values[i])) + { + diff_unequal(result, it.value(), *target_values[i], detail::concat(path, '/', detail::escape(it.key())), depth + 1); + } } else { @@ -7154,10 +7528,8 @@ public: } } - // append the "add" ops for brand-new keys collected by - // diff_object_keys -- no second source.find() per target - // key needed - result.insert(result.end(), added_ops.begin(), added_ops.end()); + // append the "add" operations for brand-new keys + diff_added(result, added, path); } break; } @@ -7230,17 +7602,17 @@ public: case value_t::object: { - std::vector common_keys; - basic_json added_ops(value_t::array); - if (diff_object_keys(result, s, t, current_path, common_keys, added_ops)) + std::vector target_values; + std::vector added; + if (diff_object_keys(result, s, t, current_path, target_values, added)) { // fast path: the frame walks source in lockstep with // common_keys, as diff_recursively does, and appends // added_ops once all members are done stack.emplace_back(&s, &t, current_path.size()); stack.back().member = s.cbegin(); - stack.back().common_keys = std::move(common_keys); - stack.back().added_ops = std::move(added_ops); + stack.back().target_values = std::move(target_values); + stack.back().added = std::move(added); } return; } @@ -7302,11 +7674,10 @@ public: if (it != s->cend()) { ++stack.back().member; - const std::size_t next_common = stack.back().next_common; - if (next_common < stack.back().common_keys.size() && it.key() == stack.back().common_keys[next_common]) + const basic_json* const target_value_ptr = stack.back().target_values[stack.back().index++]; + if (target_value_ptr != nullptr) { - ++stack.back().next_common; - const basic_json& target_value = (*t)[it.key()]; + const basic_json& target_value = *target_value_ptr; detail::concat_into(current_path, '/', detail::escape(it.key())); enter(it.value(), target_value); if (stack.size() == depth) @@ -7324,7 +7695,7 @@ public: // append the "add" ops for brand-new keys collected when the // object was entered - result.insert(result.end(), stack.back().added_ops.begin(), stack.back().added_ops.end()); + diff_added(result, stack.back().added, current_path); } // this array or object is done: continue with the one it is in diff --git a/include/nlohmann/ordered_map.hpp b/include/nlohmann/ordered_map.hpp index fdcf4a5c0..10fc764ea 100644 --- a/include/nlohmann/ordered_map.hpp +++ b/include/nlohmann/ordered_map.hpp @@ -90,6 +90,15 @@ private: return self.end(); } + /// @brief throw the exception of a failed at() lookup + /// @note kept out of line so that at_impl() stays small enough to inline + /// @throw std::out_of_range always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_key_not_found() + { + JSON_THROW(std::out_of_range("key not found")); + } + /// @brief shared implementation of the const and non-const at() overloads /// @throw std::out_of_range if @a key is not found template @@ -98,7 +107,7 @@ private: const auto it = find_impl(self, key); if (it == self.end()) { - JSON_THROW(std::out_of_range("key not found")); + throw_key_not_found(); } return it->second; } @@ -143,6 +152,17 @@ public: return {std::prev(this->end()), true}; } + /// @brief add an element at the end without checking whether the key exists + /// @note The caller must guarantee that @a key is not yet contained, otherwise + /// the container holds duplicate keys. This avoids the linear search of + /// emplace() and operator[] when many keys known to be new are added. + template::value, int> = 0> + void emplace_new_key(const key_type& key, V && t) + { + append(key, std::forward(t)); + } + T& operator[](const key_type& key) { return emplace(key, T{}).first->second; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index ccd99cfb2..e009f6545 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -3346,6 +3346,34 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept } #endif +/*! +@brief whether a value of the given type owns heap memory + +Strings, binaries, objects, and arrays are stored as pointers to allocated +values; all other types are stored directly in the union. Every enumerator is +listed so that -Wswitch-enum flags this function when a new type is added. +*/ +inline bool owns_memory(const value_t t) noexcept +{ + switch (t) + { + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::binary: + return true; + + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + return false; + } +} + /*! @brief compare an integer with a floating point number without precision loss @@ -3423,6 +3451,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // size_t +#include // move // #include @@ -3439,39 +3468,46 @@ namespace detail * Note the order of escaping "~" to "~0" and "/" to "~1" is important. * * The string is rebuilt in a single pass, appending whole runs between the - * characters that need escaping. Scanning with find_first_of() keeps the - * common case -- nothing to escape -- as fast as a single search, while - * repeated replace() calls would move the tail of the string once per - * escaped character. + * characters that need escaping, while repeated replace() calls would move + * the tail of the string once per escaped character. @ref escape_into appends + * the escaped string to an existing one. */ +template +inline void escape_into(StringType& out, const StringType& s) +{ + const std::size_t size = s.size(); + std::size_t run = 0; + for (std::size_t i = 0; i < size; ++i) + { + const char c = s[i]; + if (c == '~' || c == '/') + { + out.append(s.data() + run, i - run); + out.append(c == '~' ? "~0" : "~1", 2); + run = i + 1; + } + } + out.append(s.data() + run, size - run); +} + template inline StringType escape(const StringType& s) { - auto next_special = [&s](std::size_t from) + // a single scan for both special characters; nothing to escape is the common case + const std::size_t size = s.size(); + std::size_t i = 0; + while (i < size && s[i] != '~' && s[i] != '/') { - const auto tilde = s.find_first_of('~', from); - const auto slash = s.find_first_of('/', from); - return tilde < slash ? tilde : slash; // npos is the largest value - }; - - auto pos = next_special(0); - if (pos == StringType::npos) + ++i; + } + if (i == size) { return s; } StringType result; - result.reserve(s.size() + 2); - - std::size_t run = 0; - while (pos != StringType::npos) - { - result.append(s.data() + run, pos - run); - result.append(s[pos] == '~' ? "~0" : "~1", 2); - run = pos + 1; - pos = next_special(run); - } - result.append(s.data() + run, s.size() - run); + result.reserve(size + 2); + escape_into(result, s); return result; } @@ -3516,7 +3552,7 @@ inline void unescape(StringType& s) pos = s.find_first_of('~', run); } result.append(s.data() + run, s.size() - run); - s = result; + s = std::move(result); } } // namespace detail @@ -3743,9 +3779,11 @@ NLOHMANN_JSON_NAMESPACE_END #include // numeric_limits +#include // map, multimap #include // char_traits #include // tuple #include // false_type, is_constructible, is_integral, is_same, true_type +#include // unordered_map, unordered_multimap #include // declval #include // vector #if defined(__cpp_lib_byte) && __cpp_lib_byte >= 201603L @@ -4695,6 +4733,22 @@ struct is_compatible_integer_type : is_compatible_integer_type_impl {}; +// whether to_json() stores an integer of type CompatibleNumberIntegerType as +// number_integer +template +struct is_compatible_number_integer_type + : is_compatible_integer_type {}; + +// if number_unsigned_t is the same type as number_integer_t, unsigned integers +// are stored as number_integer as well (#2573) +template +struct is_compatible_number_integer_type, + std::is_integral, + std::is_unsigned, + negation>>::value>> + : std::is_constructible {}; + template struct is_compatible_type_impl: std::false_type {}; @@ -4909,6 +4963,31 @@ struct has_capacity : std::integral_constant struct is_ordered_map : has_capacity {}; +// whether T is a specialization of nlohmann::ordered_map, which only ever +// appends members, so they can only move when its capacity changes +template +struct is_nlohmann_ordered_map : std::false_type {}; + +template +struct is_nlohmann_ordered_map> : std::true_type {}; + +// whether adding or erasing elements never moves the other elements of the +// container T; only node-based standard containers are known to guarantee this +template +struct has_stable_elements : std::false_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + +template +struct has_stable_elements> : std::true_type {}; + // to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > T conditional_static_cast(U value) @@ -5032,6 +5111,25 @@ constexpr bool value_in_range_of(T val) template using bool_constant = std::integral_constant; +// Stands in for number_unsigned_t in overload sets if it is the same type as +// number_integer_t, so that the unsigned overloads do not collide with the +// integer ones. Never constructed. Such a basic_json stores every integer as +// number_integer (#2573). +struct same_as_number_integer_type {}; + +// NumberUnsignedType, or same_as_number_integer_type if it is the same type as +// NumberIntegerType; basic_json uses this one, as it cannot name its own member +// types through an incomplete basic_json (MSVC reports a recursive alias, and +// old GCC versions crash) +template +using number_unsigned_overload_type = typename std::conditional::value, + same_as_number_integer_type, NumberUnsignedType>::type; + +// number_unsigned_t of BasicJsonType, or same_as_number_integer_type if it is +// the same type as number_integer_t +template +using number_unsigned_overload_t = number_unsigned_overload_type; + /////////////////////////////////////////////////////////////////////////////// // is_c_string /////////////////////////////////////////////////////////////////////////////// @@ -5637,7 +5735,7 @@ void templated_json_throw(ExceptionType exception) @throw type_error.302 always */ template -JSON_HEDLEY_NO_RETURN inline void throw_type_must_be(const char* expected, const BasicJsonType& j) +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_type_must_be(const char* expected, const BasicJsonType& j) { static_cast(expected); // unused when JSON_NOEXCEPTION is defined static_cast(j); @@ -5652,7 +5750,7 @@ JSON_HEDLEY_NO_RETURN inline void throw_type_must_be(const char* expected, const @throw type_error always */ template -JSON_HEDLEY_NO_RETURN inline void throw_cannot_use_with(const exception_id id_, const char* operation, const BasicJsonType& j) +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_cannot_use_with(const exception_id id_, const char* operation, const BasicJsonType& j) { static_cast(id_); // unused when JSON_NOEXCEPTION is defined static_cast(operation); @@ -5660,6 +5758,77 @@ JSON_HEDLEY_NO_RETURN inline void throw_cannot_use_with(const exception_id id_, JSON_THROW(type_error::create(id_, concat("cannot use ", operation, " with ", j.type_name()), &j)); } +/*! +@brief throws ExceptionType with a fixed message; kept out of line so that callers do not carry the exception construction +*/ +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_literal(const exception_id id_, const char* what_arg, BasicJsonContext context) +{ + static_cast(id_); // unused when JSON_NOEXCEPTION is defined + static_cast(what_arg); + static_cast(context); + JSON_THROW(ExceptionType::create(id_, what_arg, context)); +} + +// JSON Patch errors; constructed out of line so that patch_inplace() does not carry them + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_add_parent(const BasicJsonType& parent) +{ + static_cast(parent); + JSON_THROW(out_of_range::create(exception_id::patch_add_parent_not_container, concat("cannot add value: the JSON Patch 'add' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_remove_parent(const BasicJsonType& parent) +{ + static_cast(parent); + JSON_THROW(out_of_range::create(exception_id::patch_remove_parent_not_container, concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_not_an_array(const BasicJsonType& j) +{ + static_cast(j); + JSON_THROW(parse_error::create(exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &j)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_missing_member(const StringType& op, const StringType& member, const bool string_type, const BasicJsonType& val) +{ + static_cast(op); + static_cast(member); + static_cast(string_type); + static_cast(val); + // context-sensitive error message + const std::string error_msg = (op == "op") ? std::string("operation") : concat("operation '", op, '\''); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat(error_msg, string_type ? " must have string member '" : " must have member '", member, "'"), &val)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_move_into_child(const StringType& from_path, const StringType& path, const BasicJsonType& result) +{ + static_cast(from_path); + static_cast(path); + static_cast(result); + JSON_THROW(out_of_range::create(exception_id::patch_move_into_child, concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_test_failed(const BasicJsonType& val) +{ + static_cast(val); + JSON_THROW(other_error::create(exception_id::patch_test_failed, concat("unsuccessful: ", val.dump()), &val)); +} + +template +JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_patch_invalid_operation(const StringType& op, const BasicJsonType& val) +{ + static_cast(op); + static_cast(val); + JSON_THROW(parse_error::create(exception_id::patch_invalid_operation, 0, concat("operation value '", op, "' is invalid"), &val)); +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -5784,7 +5953,11 @@ void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) { case value_t::number_unsigned: { - val = static_cast(*j.template get_ptr()); + // never null: a basic_json whose number_unsigned_t is number_integer_t + // stores no number_unsigned values (#2573) + const auto* ptr = j.template get_ptr(); + JSON_ASSERT(ptr != nullptr); + val = static_cast(*ptr); break; } case value_t::number_integer: @@ -5854,7 +6027,7 @@ inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_flo } template -inline void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +inline void from_json(const BasicJsonType& j, number_unsigned_overload_t& val) { get_arithmetic_value(j, val); } @@ -5970,6 +6143,22 @@ template void from_json_array_reserve(ConstructibleArrayType& /*arr*/, std::size_t /*size*/, priority_tag<0> /*unused*/) {} +// append through push_back() where available; containers without it (such as +// std::set) fall back to insert(end(), ...), which is what std::inserter did +template +auto from_json_array_append(ConstructibleArrayType& arr, ValueType&& value, priority_tag<1> /*unused*/) +-> decltype(arr.push_back(std::forward(value)), void()) +{ + arr.push_back(std::forward(value)); +} + +template +void from_json_array_append(ConstructibleArrayType& arr, ValueType&& value, priority_tag<0> /*unused*/) +{ + using std::end; + arr.insert(end(arr), std::forward(value)); +} + template::value, @@ -5979,17 +6168,15 @@ auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, p j.template get(), void()) { - using std::end; - ConstructibleArrayType ret; - from_json_array_reserve(ret, j.size(), priority_tag<1> {}); - std::transform(j.begin(), j.end(), - std::inserter(ret, end(ret)), [](const BasicJsonType & i) + const auto* inner_array = j.template get_ptr(); + from_json_array_reserve(ret, inner_array->size(), priority_tag<1> {}); + for (const auto& elem : *inner_array) { // get() returns *this, this won't call a from_json // method when value_type is BasicJsonType - return i.template get(); - }); + from_json_array_append(ret, elem.template get(), priority_tag<1> {}); + } arr = std::move(ret); } @@ -6075,6 +6262,23 @@ template inline void from_json_object_reserve(ConstructibleObjectType& /*obj*/, std::size_t /*size*/, priority_tag<0> /*unused*/) {} +// the keys of the source are sorted for the usual object types, so hinting the +// end makes every insertion constant-time (for containers that order their keys +// the same way); containers without emplace_hint() (such as ordered_map) fall +// back to emplace() +template +auto from_json_object_emplace(ConstructibleObjectType& obj, Key&& key, Value&& value, priority_tag<1> /*unused*/) +-> decltype(obj.emplace_hint(obj.end(), std::forward(key), std::forward(value)), void()) +{ + obj.emplace_hint(obj.end(), std::forward(key), std::forward(value)); +} + +template +void from_json_object_emplace(ConstructibleObjectType& obj, Key&& key, Value&& value, priority_tag<0> /*unused*/) +{ + obj.emplace(std::forward(key), std::forward(value)); +} + template::value, int> = 0> inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) @@ -6089,7 +6293,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) from_json_object_reserve(ret, inner_object->size(), priority_tag<1> {}); for (const auto& p : *inner_object) { - ret.emplace(p.first, p.second.template get()); + from_json_object_emplace(ret, p.first, p.second.template get(), priority_tag<1> {}); } obj = std::move(ret); } @@ -6110,7 +6314,11 @@ inline void from_json(const BasicJsonType& j, ArithmeticType& val) { case value_t::number_unsigned: { - val = static_cast(*j.template get_ptr()); + // never null: a basic_json whose number_unsigned_t is number_integer_t + // stores no number_unsigned values (#2573) + const auto* ptr = j.template get_ptr(); + JSON_ASSERT(ptr != nullptr); + val = static_cast(*ptr); break; } case value_t::number_integer: @@ -7122,6 +7330,17 @@ struct external_constructor j.replace_value(value_t::array, value); } + /// an rvalue container: move its elements + template + static void construct_moving(BasicJsonType& j, CompatibleArrayType& arr) + { + using std::begin; + using std::end; + + const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(arr)), std::make_move_iterator(end(arr)))); + j.replace_value(value_t::array, value); + } + template static void construct(BasicJsonType& j, const std::vector& arr) { @@ -7172,6 +7391,17 @@ struct external_constructor j.replace_value(value_t::object, value); } + /// an rvalue container: move its values + template + static void construct_moving(BasicJsonType& j, CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + const typename BasicJsonType::json_value value(j.template create(std::make_move_iterator(begin(obj)), std::make_move_iterator(end(obj)))); + j.replace_value(value_t::object, value); + } + template < typename BasicJsonType, typename CompatibleObjectType, enable_if_t < !std::is_same::value, int > = 0 > static void construct(BasicJsonType& j, const CompatibleObjectType& obj) @@ -7258,14 +7488,14 @@ inline void to_json(BasicJsonType& j, FloatType val) noexcept } template::value, int> = 0> + enable_if_t, CompatibleNumberUnsignedType>::value, int> = 0> inline void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept { external_constructor::construct(j, static_cast(val)); } template::value, int> = 0> + enable_if_t::value, int> = 0> inline void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept { external_constructor::construct(j, static_cast(val)); @@ -7309,6 +7539,66 @@ inline void to_json(BasicJsonType& j, const CompatibleArrayType& arr) external_constructor::construct(j, arr); } +// types that have a dedicated to_json overload and must not be treated as generic containers +template +struct has_dedicated_to_json : std::false_type {}; + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template<> +struct has_dedicated_to_json : std::true_type {}; +#endif + +#if defined(__cpp_lib_char8_t) +template +struct has_dedicated_to_json> : std::true_type {}; +#endif + +#if JSON_HAS_RANGE_VIEW_CONVERSION +template +struct is_range_view_type : is_compatible_range_view {}; +#else +template +struct is_range_view_type : std::false_type {}; +#endif + +#if JSON_USE_OBJECTS_FOR_ENUM_KEYED_MAPS +template +struct is_enum_keyed_map_type : is_enum_keyed_map {}; +#else +template +struct is_enum_keyed_map_type : std::false_type {}; +#endif + +// rvalue (non-const) container: the generic to_json takes its elements by move +template +using is_movable_container_rvalue = negation, + std::is_const, + has_dedicated_to_json, + is_basic_json, + is_range_view_type, + is_enum_keyed_map_type, + std::is_same>>>; + +// rvalue array-like containers +template +using is_rvalue_array_to_json = conjunction, + is_compatible_array_type, + negation, + std::is_same, + is_compatible_object_type, + is_compatible_string_type, + is_compatible_binary_type>>>; + +// rvalue containers: their elements are moved instead of copied +template::value, int> = 0> +inline void to_json(BasicJsonType& j, CompatibleArrayType && arr) +{ + external_constructor::construct_moving(j, arr); +} + #if JSON_HAS_RANGE_VIEW_CONVERSION template < typename BasicJsonType, typename T, enable_if_t < is_compatible_range_view>::value @@ -7382,6 +7672,19 @@ inline void to_json(BasicJsonType& j, const EnumKeyedMap& map) } #endif +// rvalue object-like containers +template +using is_rvalue_object_to_json = conjunction, + negation>, + is_compatible_object_type>; + +template::value, int> = 0> +inline void to_json(BasicJsonType& j, CompatibleObjectType && obj) +{ + external_constructor::construct_moving(j, obj); +} + template inline void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) { @@ -9378,7 +9681,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // memcpy #include // numeric_limits #include // string -#include // conditional, integral_constant, true_type, false_type +#include // conditional, integral_constant, is_signed, true_type, false_type #include // move // #include @@ -9788,6 +10091,20 @@ NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { +// whether an integer of a signed type is negative; the overload for other types +// avoids comparing an unsigned value with zero +template +constexpr bool is_negative_integer(NumberType x, std::true_type /*is_signed*/) noexcept +{ + return x < 0; +} + +template +constexpr bool is_negative_integer(NumberType /*unused*/, std::false_type /*is_signed*/) noexcept +{ + return false; +} + /*! @brief fast integer parser for an already-validated unsigned integer @@ -9821,8 +10138,10 @@ bool parse_integer_unsigned(const char* first, const char* last, NumberUnsignedT x = (x * 10u) + digit; } value = static_cast(x); - // reject values that do not round-trip into a narrower NumberUnsignedType - return static_cast(value) == x; + // reject values that do not round-trip into a narrower NumberUnsignedType, + // and values that wrap around to negative numbers if it is a signed type + // (number_unsigned_t may be the same type as number_integer_t, #2573) + return static_cast(value) == x && !is_negative_integer(value, std::is_signed()); } /*! @@ -10482,7 +10801,7 @@ inline void float_from_bits(std::uint64_t bits, float& value) noexcept { static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits"); const auto bits32 = static_cast(bits); - std::memcpy(&value, &bits32, sizeof(value)); + std::memcpy(&value, &bits32, sizeof(bits32)); } /// the powers of ten that are exact in binary64 (up to 10^22) @@ -14376,12 +14695,19 @@ Shared by json_sax_dom_parser and json_sax_dom_callback_parser. @param[in] ref the object or array that was just created @return whether parsing should continue (false after reporting out_of_range.408) */ +template +JSON_HEDLEY_NEVER_INLINE +bool container_too_large_error(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref) +{ + return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref)); +} + template bool check_container_size(SAX& sax, std::size_t len, const char* kind, BasicJsonType* ref) { if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref->max_size())) { - return sax.parse_error(0, "", out_of_range::create(exception_id::container_too_large, concat("excessive ", kind, " size: ", std::to_string(len)), ref)); + return container_too_large_error(sax, len, kind, ref); } return true; } @@ -14863,16 +15189,19 @@ class json_sax_dom_callback_parser if (keep && ref_stack.back()) { auto& obj = *ref_stack.back()->m_data.m_value.object; - const auto it = obj.find(val); - if (it != obj.end()) + // a single lookup: insert the placeholder, or find the existing member + const auto inserted = obj.emplace(val, discarded); + const auto it = inserted.first; + if (!inserted.second) { // this is a duplicate key (legal in JSON); remember its // current value so it can be restored later if the new // value is rejected by the callback, instead of being // erased together with the discarded placeholder duplicate_key_stash.emplace_back(&(it->second), it->second); + it->second = discarded; } - object_element = &(obj[val] = discarded); + object_element = &(it->second); } return true; @@ -15673,7 +16002,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current != char_traits::eof())) { - return last_byte_error(exception_id::unexpected_end_of_input, concat("expected end of input; last byte: 0x", get_token_string()), "value"); + return last_byte_error(exception_id::unexpected_end_of_input, "expected end of input", "value", ""); } } @@ -15795,7 +16124,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast(document_size) != chars_read - document_start)) { - return last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(chars_read - document_start), ")"), "document"); + return bson_document_size_error(document_size, chars_read - document_start); } return true; } @@ -15987,7 +16316,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(len < 1)) { - return last_byte_error(exception_id::unexpected_byte, concat("string length must be at least 1, is ", std::to_string(len)), "string"); + return bson_length_error("string length must be at least 1, is ", len, "string"); } if (JSON_HEDLEY_UNLIKELY(!get_string(len - static_cast(1), result))) @@ -16017,7 +16346,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(len < 0)) { - return last_byte_error(exception_id::unexpected_byte, concat("byte array length cannot be negative, is ", std::to_string(len)), "binary"); + return bson_length_error("byte array length cannot be negative, is ", len, "binary"); } // All BSON binary values have a subtype @@ -16107,9 +16436,7 @@ class binary_reader default: // anything else is not supported (yet) { - const std::string cr_str = hex_byte(static_cast(element_type)); - return sax->parse_error(element_type_parse_position, cr_str, - parse_error::create(exception_id::bson_unsupported_type, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + return bson_unsupported_type_error(element_type_parse_position, element_type); } } } @@ -16538,6 +16865,7 @@ class binary_reader @param[in] context parsing context for the error message @return whether the SAX consumer accepts the parse error */ + JSON_HEDLEY_NEVER_INLINE bool cbor_indefinite_string_error(const char* type_name, const char* context) { return last_byte_error(exception_id::invalid_string_or_size, concat("indefinite-length ", type_name, " is not allowed inside indefinite-length ", type_name, "; last byte: 0x", get_token_string()), context); @@ -16617,7 +16945,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x60-0x7B)", inside_indefinite ? "" : " or indefinite string type (0x7F)", "; last byte: 0x", get_token_string()), "string"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x60-0x7B)", "string", inside_indefinite ? "" : " or indefinite string type (0x7F)"); } } } @@ -16747,7 +17075,7 @@ class binary_reader break; } - return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key"); + return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found); } /*! @@ -16828,7 +17156,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0x40-0x5B)", inside_indefinite ? "" : " or indefinite binary array type (0x5F)", "; last byte: 0x", get_token_string()), "binary"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0x40-0x5B)", "binary", inside_indefinite ? "" : " or indefinite binary array type (0x5F)"); } } } @@ -16974,8 +17302,7 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(len) || len == detail::unknown_size())) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message(concat("excessive ", context, " size"), "size"), nullptr)); + return container_size_error(context); } result = conditional_static_cast(len); return true; @@ -17543,7 +17870,7 @@ class binary_reader default: { - return last_byte_error(exception_id::invalid_string_or_size, concat("expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x", get_token_string()), "string"); + return last_byte_error(exception_id::invalid_string_or_size, "expected length specification (0xA0-0xBF, 0xD9-0xDB)", "string", ""); } } } @@ -17635,7 +17962,7 @@ class binary_reader break; } - return last_byte_error(exception_id::invalid_string_or_size, concat("only string keys are supported, but found ", found, "; last byte: 0x", get_token_string()), "object key"); + return last_byte_error(exception_id::invalid_string_or_size, "only string keys are supported, but found ", "object key", found); } /*! @@ -18133,8 +18460,7 @@ class binary_reader } if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(number))) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("integer value overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char return true; @@ -18231,8 +18557,7 @@ class binary_reader } if (!value_in_range_of(number)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("integer value overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "integer value overflow", "size"); } result = detail::conditional_static_cast(number); return true; @@ -18310,7 +18635,7 @@ class binary_reader // or SIZE_MAX. if (JSON_HEDLEY_UNLIKELY(result > (std::numeric_limits::max)() / i)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size"); } result *= i; // the pre-check above already rules out result becoming 0 @@ -18319,7 +18644,7 @@ class binary_reader // unknown-size container (see get_ubjson_size_type()) if (result == npos) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, exception_message("excessive ndarray size caused overflow", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive ndarray size caused overflow", "size"); } if (JSON_HEDLEY_UNLIKELY(!emit_unsigned(i))) { @@ -18369,7 +18694,7 @@ class binary_reader if (input_format == input_format_t::bjdata && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second))) { - return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type"); + return optimized_type_error(); } if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type"))) @@ -18384,7 +18709,7 @@ class binary_reader { return false; } - return last_byte_error(exception_id::unexpected_byte, concat("expected '#' after type information; last byte: 0x", get_token_string()), "size"); + return last_byte_error(exception_id::unexpected_byte, "expected '#' after type information", "size", ""); } const bool is_error = get_ubjson_size_value(result.first, is_ndarray, 0, result.second); @@ -18535,7 +18860,7 @@ class binary_reader } if (JSON_HEDLEY_UNLIKELY(current > 127)) { - return last_byte_error(exception_id::invalid_string_or_size, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", get_token_string()), "char"); + return last_byte_error(exception_id::invalid_string_or_size, "byte after 'C' must be in range 0x00..0x7F", "char", ""); } string_t s(1, static_cast(current)); return sax->string(s); @@ -18619,8 +18944,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') && size_and_type.first > max_valueless_container_size)) { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, - exception_message("excessive array size", "size"), nullptr)); + return out_of_range_error(exception_id::container_too_large, "excessive array size", "size"); } if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second))) @@ -19332,7 +19656,7 @@ class binary_reader { // in case of failure, advance position by 1 to report the failing location ++chars_read; - sax->parse_error(chars_read, "", parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + static_cast(report_eof(context)); return false; } return true; @@ -19485,8 +19809,7 @@ class binary_reader const auto result = static_cast(number); if (JSON_HEDLEY_UNLIKELY(std::isfinite(number) && !std::isfinite(result))) { - return sax->parse_error(chars_read, get_token_string(), - out_of_range::create(exception_id::number_overflow, exception_message("number overflow", "value"), nullptr)); + return out_of_range_error(exception_id::number_overflow, "number overflow", "value"); } return sax->number_float(result, ""); } @@ -19701,22 +20024,82 @@ class binary_reader { if (JSON_HEDLEY_UNLIKELY(current == char_traits::eof())) { - return sax->parse_error(chars_read, "", - parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + return report_eof(context); } return true; } + /*! + @brief reports an unexpected end of input (out of line to keep the exception + construction out of the hot reading functions) + @param[in] context further context information (for diagnostics) + @return the result of the SAX parser's parse_error() + */ + JSON_HEDLEY_NON_NULL(2) + JSON_HEDLEY_NEVER_INLINE + bool report_eof(const char* context) const + { + return sax->parse_error(chars_read, "", + parse_error::create(exception_id::unexpected_end_of_input, chars_read, exception_message("unexpected end of input", context), nullptr)); + } + + /*! + @brief reports an out_of_range error at the last read byte (out of line) + */ + JSON_HEDLEY_NEVER_INLINE + bool out_of_range_error(const exception_id id_, const char* detail, const char* context) const + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(id_, exception_message(detail, context), nullptr)); + } + + JSON_HEDLEY_NEVER_INLINE + bool container_size_error(const char* what) const + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(exception_id::container_too_large, + exception_message(concat("excessive ", what, " size"), "size"), nullptr)); + } + + JSON_HEDLEY_NEVER_INLINE + bool optimized_type_error() const + { + return last_byte_error(exception_id::unexpected_byte, concat("marker 0x", get_token_string(), " is not a permitted optimized array type"), "type"); + } + + template + JSON_HEDLEY_NEVER_INLINE + bool bson_length_error(const char* prefix, const NumberType len, const char* context) const + { + return last_byte_error(exception_id::unexpected_byte, concat(prefix, std::to_string(len)), context); + } + + JSON_HEDLEY_NEVER_INLINE + bool bson_document_size_error(const std::int32_t document_size, const std::size_t bytes_read) const + { + return last_byte_error(exception_id::unexpected_byte, concat("document size ", std::to_string(document_size), " does not match the number of bytes read (", std::to_string(bytes_read), ")"), "document"); + } + + JSON_HEDLEY_NEVER_INLINE + bool bson_unsupported_type_error(const std::size_t position, const char element_type) const + { + const std::string cr_str = hex_byte(static_cast(element_type)); + return sax->parse_error(position, cr_str, + parse_error::create(exception_id::bson_unsupported_type, position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + } + /*! @brief reports a parse error at the last read byte @param[in] id_ the id of the parse_error exception @param[in] detail a detailed error message @param[in] context further context information + @param[in] tail optional text appended to the detail, followed by "; last byte: 0x" @return the result of the SAX parser's parse_error() */ - bool last_byte_error(const exception_id id_, const std::string& detail, const std::string& context) const + JSON_HEDLEY_NEVER_INLINE + bool last_byte_error(const exception_id id_, const std::string& detail, const char* context, const char* tail = nullptr) const { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, exception_message(detail, context), nullptr)); + // with a tail, the message reads "; last byte: 0x" + return sax->parse_error(chars_read, get_token_string(), parse_error::create(id_, chars_read, + exception_message(tail == nullptr ? detail : concat(detail, tail, "; last byte: 0x", get_token_string()), context), nullptr)); } /*! @@ -19725,7 +20108,8 @@ class binary_reader @param[in] context further context information @return the result of the SAX parser's parse_error() */ - bool unexpected_byte(const char* detail, const std::string& context) const + JSON_HEDLEY_NEVER_INLINE + bool unexpected_byte(const char* detail, const char* context) const { return last_byte_error(exception_id::unexpected_byte, concat(detail, ": 0x", get_token_string()), context); } @@ -19736,6 +20120,7 @@ class binary_reader @param[in] context further context information @return the result of the SAX parser's parse_error() */ + JSON_HEDLEY_NEVER_INLINE bool length_type_error(const char* position, const char* context) const { const char* types = input_format == input_format_t::bjdata ? "U, i, u, I, m, l, M, L" : "U, i, I, l, L"; @@ -20974,7 +21359,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci } /// @throw invalid_iterator.214 always - JSON_HEDLEY_NO_RETURN void throw_cannot_get_value() const + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE void throw_cannot_get_value() const { JSON_THROW(invalid_iterator::create(exception_id::iterator_value_unavailable, "cannot get value", m_object)); } @@ -20988,40 +21373,25 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); - return m_it.object_iterator->second; - } - - case value_t::array: - { - JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); - return *m_it.array_iterator; - } - - case value_t::null: - throw_cannot_get_value(); - - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) - { - return *m_object; - } - - throw_cannot_get_value(); - } + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); + return *m_it.array_iterator; } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); + return m_it.object_iterator->second; + } + + // neither array nor object: a primitive value (null has no value to point to) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type != value_t::null && m_it.primitive_iterator.is_begin())) + { + return *m_object; + } + + throw_cannot_get_value(); } /*! @@ -21032,38 +21402,25 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); - return &(m_it.object_iterator->second); - } - - case value_t::array: - { - JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); - return &*m_it.array_iterator; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) - { - return m_object; - } - - throw_cannot_get_value(); - } + JSON_ASSERT(m_it.array_iterator != m_object->m_data.m_value.array->end()); + return &*m_it.array_iterator; } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + JSON_ASSERT(m_it.object_iterator != m_object->m_data.m_value.object->end()); + return &(m_it.object_iterator->second); + } + + // neither array nor object: a primitive value + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return m_object; + } + + throw_cannot_get_value(); } /*! @@ -21085,35 +21442,20 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - std::advance(m_it.object_iterator, 1); - break; - } - - case value_t::array: - { - std::advance(m_it.array_iterator, 1); - break; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - ++m_it.primitive_iterator; - break; - } + ++m_it.array_iterator; + return *this; } + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + std::advance(m_it.object_iterator, 1); + return *this; + } + + // neither array nor object: a primitive value + ++m_it.primitive_iterator; return *this; } @@ -21136,35 +21478,20 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci { JSON_ASSERT(m_object != nullptr); - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - { - std::advance(m_it.object_iterator, -1); - break; - } - - case value_t::array: - { - std::advance(m_it.array_iterator, -1); - break; - } - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - { - --m_it.primitive_iterator; - break; - } + --m_it.array_iterator; + return *this; } + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + std::advance(m_it.object_iterator, -1); + return *this; + } + + // neither array nor object: a primitive value + --m_it.primitive_iterator; return *this; } @@ -21178,7 +21505,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci // if objects are not the same, the comparison is undefined if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) { - JSON_THROW(invalid_iterator::create(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object)); + throw_literal(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object); } // value-initialized forward iterators can be compared, and must compare equal to other value-initialized iterators of the same type #4493 @@ -21187,25 +21514,18 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci return true; } - switch (m_object->m_data.m_type) + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::array)) { - case value_t::object: - return (m_it.object_iterator == other.m_it.object_iterator); - - case value_t::array: - return (m_it.array_iterator == other.m_it.array_iterator); - - case value_t::null: - case value_t::string: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::binary: - case value_t::discarded: - default: - return (m_it.primitive_iterator == other.m_it.primitive_iterator); + return (m_it.array_iterator == other.m_it.array_iterator); } + + if (JSON_HEDLEY_LIKELY(m_object->m_data.m_type == value_t::object)) + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + // neither array nor object: a primitive value + return (m_it.primitive_iterator == other.m_it.primitive_iterator); } /*! @@ -21227,7 +21547,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci // if objects are not the same, the comparison is undefined if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) { - JSON_THROW(invalid_iterator::create(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object)); + throw_literal(exception_id::iterators_compare_different_values, "cannot compare iterators of different containers", m_object); } // value-initialized forward iterators can be compared, and must compare equal to other value-initialized iterators of the same type #4493 @@ -21240,7 +21560,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_order_on_object, "cannot compare order of object iterators", m_object)); + throw_literal(exception_id::iterator_order_on_object, "cannot compare order of object iterators", m_object); case value_t::array: return (m_it.array_iterator < other.m_it.array_iterator); @@ -21296,7 +21616,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object)); + throw_literal(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object); case value_t::array: { @@ -21375,7 +21695,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object)); + throw_literal(exception_id::iterator_arithmetic_on_object, "cannot use offsets with object iterators", m_object); case value_t::array: return m_it.array_iterator - other.m_it.array_iterator; @@ -21404,7 +21724,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci switch (m_object->m_data.m_type) { case value_t::object: - JSON_THROW(invalid_iterator::create(exception_id::iterator_subscript_on_object, "cannot use operator[] for object iterators", m_object)); + throw_literal(exception_id::iterator_subscript_on_object, "cannot use operator[] for object iterators", m_object); case value_t::array: return *std::next(m_it.array_iterator, n); @@ -21444,7 +21764,7 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci return m_it.object_iterator->first; } - JSON_THROW(invalid_iterator::create(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object)); + throw_literal(exception_id::iterator_key_not_object, "cannot use key() for non-object iterators", m_object); } /*! @@ -21676,8 +21996,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // ostream #endif // JSON_NO_IO #include // max -#include // map -#include // accumulate #include // string #include // move #include // vector @@ -21686,6 +22004,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -21735,12 +22055,21 @@ class json_pointer /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ string_t to_string() const { - return std::accumulate(reference_tokens.begin(), reference_tokens.end(), - string_t{}, - [](const string_t& a, const string_t& b) + // one buffer: every token adds its escaped form (at least its length) and a slash + std::size_t length = 0; + for (const auto& reference_token : reference_tokens) { - return detail::concat(a, '/', detail::escape(b)); - }); + length += reference_token.size() + 1; + } + + string_t result; + result.reserve(length); + for (const auto& reference_token : reference_tokens) + { + result.append("/", 1); + detail::escape_into(result, reference_token); + } + return result; } /// @brief return a string representation of the JSON pointer @@ -21795,21 +22124,29 @@ class json_pointer friend json_pointer operator/(const json_pointer& lhs, const json_pointer& rhs) { - return json_pointer(lhs) /= rhs; + json_pointer result; + result.reference_tokens.reserve(lhs.reference_tokens.size() + rhs.reference_tokens.size()); + result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); + result.reference_tokens.insert(result.reference_tokens.end(), rhs.reference_tokens.begin(), rhs.reference_tokens.end()); + return result; } /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ friend json_pointer operator/(const json_pointer& lhs, string_t token) // NOLINT(performance-unnecessary-value-param) { - return json_pointer(lhs) /= std::move(token); + json_pointer result; + result.reference_tokens.reserve(lhs.reference_tokens.size() + 1); + result.reference_tokens.insert(result.reference_tokens.end(), lhs.reference_tokens.begin(), lhs.reference_tokens.end()); + result.reference_tokens.push_back(std::move(token)); + return result; } /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx) { - return json_pointer(lhs) /= array_idx; + return lhs / detail::to_string(array_idx); } /// @brief returns the parent of this JSON pointer @@ -21821,8 +22158,8 @@ class json_pointer return *this; } - json_pointer res = *this; - res.pop_back(); + json_pointer res; + res.reference_tokens = std::vector(reference_tokens.begin(), reference_tokens.end() - 1); return res; } @@ -21911,14 +22248,14 @@ class json_pointer private: /// @throw out_of_range.405 always - JSON_HEDLEY_NO_RETURN static void throw_no_parent() + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_no_parent() { JSON_THROW(detail::out_of_range::create(detail::exception_id::patch_on_root, "JSON pointer has no parent", nullptr)); } /// @throw out_of_range.404 always template - JSON_HEDLEY_NO_RETURN static void throw_unresolved(const string_t& reference_token, BasicJsonContext context) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE static void throw_unresolved(const string_t& reference_token, BasicJsonContext context) { static_cast(reference_token); // unused when JSON_NOEXCEPTION is defined static_cast(context); @@ -21971,6 +22308,28 @@ class json_pointer return array_index_status::not_a_number; } + // fast path: up to 18 decimal digits cannot overflow unsigned long long or + // size_type (9 digits for a 32-bit size_type), so strtoull and errno are not needed + constexpr std::size_t max_fast_digits = sizeof(size_type) >= 8 ? 18 : 9; + if (!s.empty() && s.size() <= max_fast_digits) + { + size_type acc = 0; + for (std::size_t i = 0; i < s.size(); ++i) + { + const char c = s[i]; + if (JSON_HEDLEY_UNLIKELY(c < '0' || c > '9')) + { + break; // not a plain number: let strtoull decide + } + acc = static_cast(acc * 10 + static_cast(c - '0')); + if (JSON_HEDLEY_UNLIKELY(i == s.size() - 1)) + { + idx = acc; + return array_index_status::ok; + } + } + } + const char* p = s.data(); char* p_end = nullptr; // NOLINT(misc-const-correctness) errno = 0; // strtoull doesn't reset errno @@ -22027,6 +22386,83 @@ class json_pointer return idx; } + /*! + @brief compare with a JSON pointer string without splitting it into tokens + + @throw parse_error.107 if the string is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ + bool equals_string(const string_t& reference_string) const + { + if (reference_string.empty()) + { + return reference_tokens.empty(); + } + + if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/')) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr)); + } + + // the whole string is validated, whether or not it matches + const std::size_t size = reference_string.size(); + for (std::size_t pos = reference_string.find_first_of('~'); + pos != string_t::npos; + pos = reference_string.find_first_of('~', pos + 1)) + { + if (JSON_HEDLEY_UNLIKELY(pos == size - 1 || (reference_string[pos + 1] != '0' && reference_string[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + } + } + + // compare token by token, unescaping the string on the fly + std::size_t pos = 0; // the slash in front of the next token + for (const auto& reference_token : reference_tokens) + { + if (pos >= size) + { + return false; + } + + std::size_t i = pos + 1; + std::size_t j = 0; + const std::size_t token_size = reference_token.size(); + while (i < size && reference_string[i] != '/') + { + char c = reference_string[i++]; + if (c == '~') + { + c = (reference_string[i++] == '0') ? '~' : '/'; + } + if (j == token_size || reference_token[j] != c) + { + return false; + } + ++j; + } + if (j != token_size) + { + return false; + } + pos = i; + } + return pos == size; + } + + /// @brief object member access with a key that is only copied if it is inserted + template + static BasicJsonType& object_member(BasicJsonType& object, const string_t& key) + { + auto& members = *object.m_data.m_value.object; + const auto it = members.find(key); + if (it != members.end()) + { + return it->second; + } + return object.operator[](key); + } + JSON_PRIVATE_UNLESS_TESTED: json_pointer top() const { @@ -22050,84 +22486,141 @@ class json_pointer the result independent of the order in which the flattened object is iterated, which is unspecified for some object types. - The prefixes form a tree and are numbered, so each of them is stored only - once (as a node) rather than as a copy of all of its reference tokens. + The prefixes form a tree and are numbered (number 0 is the empty prefix), + so each of them is stored only once (as a node) rather than as a copy of + all of its reference tokens. The tree does not own the reference tokens: + it refers to the tokens of the parsed keys, which must outlive it. */ struct prefix_tree { - // children[id] maps a reference token to the number of the prefix - // extended by that token; number 0 is the empty prefix - std::vector> children; + // open-addressing hash table from (parent, last reference token) to + // the number of the node; empty slots have token == nullptr + struct slot + { + std::size_t hash; + std::size_t parent; + const string_t* token; + std::size_t id; + + slot() noexcept + : hash(0), parent(0), token(nullptr), id(0) + {} + + slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept + : hash(h), parent(p), token(t), id(i) + {} + }; + + std::vector slots; + std::size_t used = 0; + + static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept + { + std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL); + for (std::size_t i = 0; i < token.size(); ++i) + { + h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL); + } + return h ^ (h >> 29u); + } + + void grow() + { + std::vector old(slots.size() * 2); + old.swap(slots); + const std::size_t mask = slots.size() - 1; + for (const auto& s : old) + { + if (s.token != nullptr) + { + std::size_t i = s.hash & mask; + while (slots[i].token != nullptr) + { + i = (i + 1) & mask; + } + slots[i] = s; + } + } + } + + // parent[id] is the number of the prefix without its last token + std::vector parent; // is_array[id] is true iff some flattened key has the reference token // 0 directly below the prefix with number id std::vector is_array; // start with the empty prefix only prefix_tree() - : children(1) + : slots(16) + , parent(1, 0) , is_array(1, false) {} // return the number of the prefix with number id extended by // reference_token, adding it if it is new - std::size_t add_child(std::size_t id, string_t&& reference_token) + std::size_t add_child(std::size_t id, const string_t& reference_token) { + const std::size_t hash = hash_of(id, reference_token); + const std::size_t mask = slots.size() - 1; + std::size_t i = hash & mask; + while (slots[i].token != nullptr) + { + if (slots[i].hash == hash && slots[i].parent == id && *slots[i].token == reference_token) + { + return slots[i].id; + } + i = (i + 1) & mask; + } + + // new prefix + const std::size_t child = parent.size(); + slots[i] = slot(hash, id, &reference_token, child); if (reference_token == "0") { is_array[id] = true; } + parent.push_back(id); + is_array.push_back(false); - // read the number before the emplace_back below, which may - // reallocate children and invalidate the iterator - const std::size_t next = children.size(); - const auto inserted = children[id].emplace(std::move(reference_token), next); - const std::size_t child = inserted.first->second; - if (inserted.second) + // keep the load factor below 1/2 + if (++used * 2 > slots.size()) { - children.emplace_back(); - is_array.push_back(false); + grow(); } return child; } - - // return the number of the prefix with number id extended by - // reference_token, which must have been added before - std::size_t find_child(std::size_t id, const string_t& reference_token) const - { - const auto it = children[id].find(reference_token); - JSON_ASSERT(it != children[id].end()); - return it->second; - } }; /*! @brief create and return a reference to the pointed to value - Complexity: Linear in the number of reference tokens (times the logarithm - of the number of siblings for the prefix lookup). + @param[in] j the value to create the pointed to value in + @param[in] tokens the reference tokens of the pointer + @param[in] prefixes prefixes[i] is the number in @a tree of the prefix + consisting of the first i reference tokens + + Complexity: Linear in the number of reference tokens. @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ template - BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const + static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector& tokens, + const std::vector& prefixes, const prefix_tree& tree) { auto* result = &j; - // the number of the prefix consumed so far; used to look up whether - // the value to be created below is an array or an object - std::size_t id = 0; - // in case no reference tokens exist, return a reference to the JSON value // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) + for (std::size_t i = 0; i < tokens.size(); ++i) { + const auto& reference_token = tokens[i]; switch (result->type()) { case detail::value_t::null: { - if (tree.is_array[id]) + if (tree.is_array[prefixes[i]]) { // some reference token below this position is 0, so the // value is an array @@ -22144,7 +22637,7 @@ class json_pointer case detail::value_t::object: { // create an entry in the object - result = &result->operator[](reference_token); + result = &object_member(*result, reference_token); break; } @@ -22171,8 +22664,6 @@ class json_pointer default: JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j)); } - - id = tree.find_child(id, reference_token); } return *result; @@ -22224,7 +22715,7 @@ class json_pointer case detail::value_t::object: { // use unchecked object access - ptr = &ptr->operator[](reference_token); + ptr = &object_member(*ptr, reference_token); break; } @@ -22274,8 +22765,13 @@ class json_pointer { case detail::value_t::object: { - // note: at performs range check - ptr = &ptr->at(reference_token); + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) + { + // at() throws out_of_range.403 with the usual message + ptr->at(reference_token); + } + ptr = &it->second; break; } @@ -22296,7 +22792,7 @@ class json_pointer JSON_THROW(detail::out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat( "array index ", std::to_string(idx), " is out of range"), ptr)); } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -22344,7 +22840,9 @@ class json_pointer { // use unchecked object access; the const operator[] // asserts that the key exists - ptr = &ptr->operator[](reference_token); + const auto it = ptr->m_data.m_value.object->find(reference_token); + JSON_ASSERT(it != ptr->m_data.m_value.object->end()); + ptr = &it->second; break; } @@ -22358,7 +22856,9 @@ class json_pointer // use unchecked array access; the const operator[] // asserts that the index exists - ptr = &ptr->operator[](array_index(reference_token)); + const auto idx = array_index(reference_token); + JSON_ASSERT(idx < ptr->m_data.m_value.array->size()); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -22399,12 +22899,12 @@ class json_pointer { case detail::value_t::object: { - const auto it = ptr->find(reference_token); - if (JSON_HEDLEY_UNLIKELY(it == ptr->end())) + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) { return nullptr; } - ptr = &*it; + ptr = &it->second; break; } @@ -22441,7 +22941,7 @@ class json_pointer { return nullptr; } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -22474,13 +22974,14 @@ class json_pointer { case detail::value_t::object: { - if (!ptr->contains(reference_token)) + const auto it = ptr->m_data.m_value.object->find(reference_token); + if (it == ptr->m_data.m_value.object->end()) { // we did not find the key in the object return false; } - ptr = &ptr->operator[](reference_token); + ptr = &it->second; break; } @@ -22502,13 +23003,13 @@ class json_pointer return false; } - if (idx >= ptr->size()) + if (idx >= ptr->m_data.m_value.array->size()) { // index out of range return false; } - ptr = &ptr->operator[](idx); + ptr = &(*ptr->m_data.m_value.array)[idx]; break; } @@ -22558,6 +23059,15 @@ class json_pointer JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", reference_string, "'"), nullptr)); } + // every slash starts a reference token + const std::size_t size = reference_string.size(); + std::size_t token_count = 0; + for (std::size_t i = 0; i < size; ++i) + { + token_count += (reference_string[i] == '/') ? 1 : 0; + } + result.reserve(token_count); + // extract the reference tokens: // - slash: position of the last read slash (or end of string) // - start: position after the previous slash @@ -22579,31 +23089,52 @@ class json_pointer const auto count = (slash == string_t::npos ? reference_string.size() : slash) - start; auto reference_token = string_t(reference_string.data() + start, count); - // check reference tokens are properly escaped - for (std::size_t pos = reference_token.find_first_of('~'); - pos != string_t::npos; - pos = reference_token.find_first_of('~', pos + 1)) + // check reference tokens are properly escaped; a token without + // a '~' (the common case) needs neither check nor unescaping + std::size_t pos = reference_token.find_first_of('~'); + if (pos != string_t::npos) { - JSON_ASSERT(reference_token[pos] == '~'); - - // ~ must be followed by 0 or 1 - if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || - (reference_token[pos + 1] != '0' && - reference_token[pos + 1] != '1'))) + for (; pos != string_t::npos; pos = reference_token.find_first_of('~', pos + 1)) { - JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + JSON_ASSERT(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || + (reference_token[pos + 1] != '0' && + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr)); + } } + + detail::unescape(reference_token); } // finally, store the reference token - detail::unescape(reference_token); - result.push_back(reference_token); + result.push_back(std::move(reference_token)); } return result; } private: + // Add the entry (key, v) to the object `obj`. Flattened paths are pairwise + // distinct, so objects that can append a key known to be absent + // (ordered_map) skip the lookup, which is linear and would make flatten + // quadratic; all other object types use operator[]. + template + static auto add_flattened(ObjectType& obj, const string_t& key, ValueType&& v, detail::priority_tag<1> /*unused*/) + -> decltype(obj.emplace_new_key(key, std::forward(v)), void()) + { + obj.emplace_new_key(key, std::forward(v)); + } + + template + static void add_flattened(ObjectType& obj, const string_t& key, ValueType&& v, detail::priority_tag<0> /*unused*/) + { + obj[key] = std::forward(v); + } + /*! @param[in] reference_string the reference string to the current value @param[in] value the value to consider @@ -22648,7 +23179,8 @@ class json_pointer // flatten `v`, whose path is `path`: primitives and empty containers // are added to the result right away; other containers get a frame - const auto enter = [&stack, &path, &result](const BasicJsonType & v) + auto& object = *result.m_data.m_value.object; + const auto enter = [&stack, &path, &object](const BasicJsonType & v) { switch (v.type()) { @@ -22657,7 +23189,7 @@ class json_pointer if (v.m_data.m_value.array->empty()) { // flatten empty array as null - result[path] = nullptr; + add_flattened(object, path, nullptr, detail::priority_tag<1> {}); } else { @@ -22671,7 +23203,7 @@ class json_pointer if (v.m_data.m_value.object->empty()) { // flatten empty object as null - result[path] = nullptr; + add_flattened(object, path, nullptr, detail::priority_tag<1> {}); } else { @@ -22691,7 +23223,7 @@ class json_pointer default: { // add a primitive value with its reference string - result[path] = v; + add_flattened(object, path, v, detail::priority_tag<1> {}); return; } } @@ -22737,6 +23269,10 @@ class json_pointer enter(it->second); } } + + // the entries were added without going through basic_json, so the + // parent pointers (JSON_DIAGNOSTICS) have to be set now + result.set_parents(); } /*! @@ -22760,21 +23296,53 @@ class json_pointer BasicJsonType result; + // parse every key once; the reference tokens are kept for the second + // pass (the vectors are never resized, so the tree may refer to them) + std::vector> keys; + keys.reserve(value.m_data.m_value.object->size()); + for (const auto& element : *value.m_data.m_value.object) + { + json_pointer ptr(element.first); + keys.push_back(std::move(ptr.reference_tokens)); + } + // collect the pointer prefixes that have a reference token 0 among // their children; the values below them are arrays, all others are // objects (see prefix_tree) prefix_tree tree; - for (const auto& element : *value.m_data.m_value.object) + // the number of the prefix with i + 1 tokens of the previous key; keys + // are often sorted, so most of the prefix is the same as for the + // previous key and need not be looked up again + std::vector path; + const std::vector* previous = nullptr; + // the number of the complete prefix (i.e., the key) of each key + std::vector leaves; + leaves.reserve(keys.size()); + for (const auto& tokens : keys) { - json_pointer ptr(element.first); - std::size_t id = 0; - for (auto& reference_token : ptr.reference_tokens) + std::size_t common = 0; + if (previous != nullptr) { - id = tree.add_child(id, std::move(reference_token)); + const std::size_t limit = (std::min)(tokens.size(), previous->size()); + while (common < limit && tokens[common] == (*previous)[common]) + { + ++common; + } } + path.resize(common); + std::size_t id = common == 0 ? 0 : path[common - 1]; + for (std::size_t i = common; i < tokens.size(); ++i) + { + id = tree.add_child(id, tokens[i]); + path.push_back(id); + } + leaves.push_back(id); + previous = &tokens; } // iterate the JSON object values + std::vector prefixes; + std::size_t key_index = 0; for (const auto& element : *value.m_data.m_value.object) { if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) @@ -22782,11 +23350,21 @@ class json_pointer JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second)); } + const auto& tokens = keys[key_index]; + // number the prefixes of this key from its last one to the first + prefixes.resize(tokens.size()); + for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i) + { + id = tree.parent[id]; + prefixes[i - 1] = id; + } + ++key_index; + // Assign the value to the reference pointed to by JSON pointer. Note // that if the JSON pointer is "" (i.e., points to the whole value), // function get_and_create returns a reference to the result itself. // An assignment will then create a primitive value. - json_pointer(element.first).get_and_create(result, tree) = element.second; + get_and_create(result, tokens, prefixes, tree) = element.second; } return result; @@ -22825,7 +23403,7 @@ class json_pointer = delete; #else { - return *this == json_pointer(rhs); + return equals_string(rhs); } #endif @@ -22909,7 +23487,7 @@ inline bool operator==(const json_pointer& lhs, = delete; #else { - return lhs == json_pointer(rhs); + return lhs.equals_string(rhs); } #endif @@ -22922,7 +23500,7 @@ inline bool operator==(const StringType& lhs, = delete; #else { - return json_pointer(lhs) == rhs; + return rhs.equals_string(lhs); } #endif @@ -23603,7 +24181,7 @@ class binary_writer case value_t::number_unsigned: { - write_cbor_head(0x00, j.m_data.m_value.number_unsigned); + write_cbor_head(0x00, conditional_static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -24983,7 +25561,7 @@ class binary_writer return calc_bson_integer_size(j.m_data.m_value.number_integer); case value_t::number_unsigned: - return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); + return calc_bson_unsigned_size(conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return calc_bson_string_size(*j.m_data.m_value.string, j); @@ -25025,7 +25603,7 @@ class binary_writer return write_bson_integer(name, j.m_data.m_value.number_integer); case value_t::number_unsigned: - return write_bson_unsigned(name, j.m_data.m_value.number_unsigned); + return write_bson_unsigned(name, conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return write_bson_string(name, *j.m_data.m_value.string); @@ -26919,7 +27497,10 @@ JSON_HEDLEY_ALWAYS_INLINE shortest_decimal to_shortest(std::uint64_t bits) noexc const uint128_parts p10 = pow10(-dec_exp - 1); const uint128_parts p = umul192_hi128(p10.high, p10.low, bin_sig << static_cast(shift)); std::uint64_t integral = p.high >> static_cast(extra_shift); - const std::uint64_t fractional = (p.high << static_cast(64 - extra_shift)) | (p.low >> static_cast(extra_shift)); + // (no casts in the shifts: MSVC 2015 warns C4554 on them despite the parentheses) + constexpr unsigned fractional_low_shift = static_cast(extra_shift); + constexpr unsigned fractional_high_shift = 64u - fractional_low_shift; + const std::uint64_t fractional = (p.high << fractional_high_shift) | (p.low >> fractional_low_shift); std::uint64_t digit = 0; bool round_up = false; bool round_down = false; @@ -28444,6 +29025,12 @@ class serializer using number_float_t = typename BasicJsonType::number_float_t; using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type of the absolute value of an integer: number_unsigned_t, or its + /// unsigned counterpart if it is signed (it may be number_integer_t, #2573) + using number_magnitude_t = typename std::conditional < + std::is_integral::value && std::is_signed::value, + std::make_unsigned, + std::enable_if>::type::type; using binary_char_t = typename BasicJsonType::binary_t::value_type; public: @@ -29167,6 +29754,33 @@ class serializer break; } } + + // A byte below 0x60 is a complete code point and needs no + // UTF-8 decoding. The scanner stops only at a control + // character, a quote, or a backslash in that range, so look up + // its escape sequence and write all 8 bytes of the table entry + // straight into the write buffer; only the length stored in the + // last byte of the entry is kept. + if (data[i] < 0x60) + { + // emit any bytes still pending in string_buffer first to + // preserve output order + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + const auto& escape = ascii_escapes()[data[i]]; + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size()); + write_buffer_pos += static_cast(escape[7]); + bytes_after_last_accept = 0; + undumped_chars = 0; + continue; + } } const auto byte = static_cast(s[i]); @@ -29175,92 +29789,39 @@ class serializer { case UTF8_ACCEPT: // decode found a new code point { - switch (codepoint) + // the code point is at least 0x60 (smaller ones are handled + // above) and is not printable ASCII (which the scanner + // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII + // character + if (EnsureAscii) { - case 0x08: // backspace + JSON_ASSERT(codepoint >= 0x7F); + if (error_handler == error_handler_t::keep) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; + // this character was buffered as raw bytes below + // in case it turned out to be part of an + // ill-formed sequence (which is kept as is); now + // that it decoded to a well-formed code point, + // undo that and \u-escape it like any other + // character instead + bytes = bytes_after_last_accept; } - - case 0x09: // horizontal tab + if (codepoint <= 0xFFFF) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; + write_u_escape(bytes, static_cast(codepoint)); } - - case 0x0A: // newline + else { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // EnsureAscii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) - { - if (EnsureAscii && error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes - // below in case it turned out to be part of - // an ill-formed sequence (which is kept as - // is); now that it decoded to a well-formed - // code point, undo that and \u-escape it - // like any other character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; + write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); + write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); } } + else + { + // copy byte to buffer (all previous bytes of the + // sequence have been copied in the default case below) + string_buffer[bytes++] = s[i]; + } // write buffer and reset index; there must be 13 bytes // left, as this is the maximal number of bytes to be @@ -29622,7 +30183,7 @@ class serializer @param[in] x unsigned integer number to count its digits @return number of decimal digits */ - unsigned int count_digits(number_unsigned_t x) noexcept + unsigned int count_digits(number_magnitude_t x) noexcept { unsigned int n_digits = 1; for (;;) @@ -29648,6 +30209,121 @@ class serializer } } + /*! + @brief the JSON form of each character below 0x60 + + Entry c holds the escape sequence of character c - the short form for + \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters - + or c itself if it needs no escaping, padded with zeros to 7 bytes. The last + byte is the length of the sequence. The 8 bytes of an entry are copied at + once, so writing an escape needs neither a branch nor a byte-wise loop. + */ + static const std::array, 0x60>& ascii_escapes() noexcept + { + static const std::array, 0x60> escapes = + { + { + {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00 + {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01 + {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02 + {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03 + {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04 + {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05 + {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06 + {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07 + {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08 + {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09 + {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A + {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B + {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C + {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D + {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E + {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F + {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10 + {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11 + {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12 + {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13 + {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14 + {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15 + {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16 + {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17 + {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18 + {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19 + {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A + {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B + {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C + {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D + {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E + {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F + {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20 + {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21 + {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22 + {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23 + {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24 + {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25 + {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26 + {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27 + {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28 + {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29 + {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A + {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B + {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C + {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D + {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E + {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F + {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30 + {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31 + {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32 + {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33 + {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34 + {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35 + {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36 + {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37 + {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38 + {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39 + {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A + {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B + {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C + {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D + {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E + {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F + {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40 + {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41 + {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42 + {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43 + {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44 + {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45 + {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46 + {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47 + {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48 + {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49 + {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A + {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B + {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C + {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D + {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E + {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F + {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50 + {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51 + {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52 + {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53 + {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54 + {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55 + {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56 + {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57 + {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58 + {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59 + {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A + {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B + {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C + {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D + {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E + {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F + } + }; + return escapes; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * @@ -29805,7 +30481,7 @@ class serializer } auto* buffer_ptr = write_buffer.data() + write_buffer_pos; - number_unsigned_t abs_value; + number_magnitude_t abs_value; // one byte for the minus sign unsigned int n_chars = 0; @@ -29818,7 +30494,7 @@ class serializer } else { - abs_value = static_cast(x); + abs_value = static_cast(x); } // up to 16 digits: eight at a time (as the digits of floats), written @@ -29986,17 +30662,6 @@ class serializer } } - /* - * Overload to make the compiler happy while it is instantiating - * dump_integer for number_unsigned_t. - * Must never be called. - */ - number_unsigned_t remove_sign(number_unsigned_t x) - { - JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE - return x; // LCOV_EXCL_LINE - } - /* * Helper function for dump_integer * @@ -30006,10 +30671,10 @@ class serializer * absolute values of INT_MIN and INT_MAX are usually not the same. See * #1708 for details. */ - number_unsigned_t remove_sign(number_integer_t x) noexcept + number_magnitude_t remove_sign(number_integer_t x) noexcept { JSON_ASSERT(x < 0); - return static_cast(-(x + 1)) + 1; + return static_cast(-(x + 1)) + 1; } private: @@ -30150,6 +30815,15 @@ private: return self.end(); } + /// @brief throw the exception of a failed at() lookup + /// @note kept out of line so that at_impl() stays small enough to inline + /// @throw std::out_of_range always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_key_not_found() + { + JSON_THROW(std::out_of_range("key not found")); + } + /// @brief shared implementation of the const and non-const at() overloads /// @throw std::out_of_range if @a key is not found template @@ -30158,7 +30832,7 @@ private: const auto it = find_impl(self, key); if (it == self.end()) { - JSON_THROW(std::out_of_range("key not found")); + throw_key_not_found(); } return it->second; } @@ -30203,6 +30877,17 @@ public: return {std::prev(this->end()), true}; } + /// @brief add an element at the end without checking whether the key exists + /// @note The caller must guarantee that @a key is not yet contained, otherwise + /// the container holds duplicate keys. This avoids the linear search of + /// emplace() and operator[] when many keys known to be new are added. + template::value, int> = 0> + void emplace_new_key(const key_type& key, V && t) + { + append(key, std::forward(t)); + } + T& operator[](const key_type& key) { return emplace(key, T{}).first->second; @@ -30893,8 +31578,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec "NumberUnsignedType must be at least as wide as NumberIntegerType, " "because it has to hold the absolute value of every NumberIntegerType value"); + // NumberUnsignedType may be the same type as NumberIntegerType (#2573), but + // a repeated NumberFloatType would give overloads that collide, so it gets + // a readable diagnostic instead. + static_assert(!std::is_same::value && !std::is_same::value, + "NumberFloatType must be a different type than NumberIntegerType and NumberUnsignedType"); + private: + /// the kind a value of kind @a t is stored as: if number_unsigned_t is the + /// same type as number_integer_t, every integer is stored as number_integer + /// (#2573); every other kind is stored as itself + static constexpr value_t stored_kind(const value_t t) noexcept + { + return (std::is_same::value && t == value_t::number_unsigned) ? value_t::number_integer : t; + } + /// helper for exception-safe object creation template JSON_HEDLEY_RETURNS_NON_NULL @@ -30968,8 +31667,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec json_value(boolean_t v) noexcept : boolean(v) {} /// constructor for numbers (integer) json_value(number_integer_t v) noexcept : number_integer(v) {} - /// constructor for numbers (unsigned) - json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (unsigned); takes a placeholder if number_unsigned_t is number_integer_t + json_value(detail::number_unsigned_overload_type v) noexcept : number_unsigned(v) {} /// constructor for numbers (floating-point) json_value(number_float_t v) noexcept : number_float(v) {} /// constructor for empty values of a given type @@ -30978,26 +31677,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec switch (t) { case value_t::object: - { - object = create(); - break; - } - case value_t::array: - { - array = create(); - break; - } - case value_t::string: - { - string = create(""); - break; - } - case value_t::binary: { - binary = create(); + // allocating: kept out of line, so the scalar cases below inline + init_owned(t); break; } @@ -31037,13 +31722,41 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec object = nullptr; // silence warning, see #821 if (JSON_HEDLEY_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(detail::exception_id::internal_error, "961c151d2e87f2686a955a9be24d316f1362bf21 3.12.0", nullptr)); // LCOV_EXCL_LINE + detail::throw_literal(detail::exception_id::internal_error, "961c151d2e87f2686a955a9be24d316f1362bf21 3.12.0", nullptr); // LCOV_EXCL_LINE } break; } } } + JSON_HEDLEY_NEVER_INLINE + void init_owned(value_t t) + { + switch (t) + { + case value_t::object: + object = create(); + break; + case value_t::array: + array = create(); + break; + case value_t::string: + string = create(""); + break; + case value_t::binary: + binary = create(); + break; + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + break; + } + } + /// constructor for strings json_value(const string_t& value) : string(create(value)) {} @@ -31113,6 +31826,34 @@ private: } } + // true if no child of the non-empty array/object v has children of + // its own, i.e. v can be freed in one go (its destructor then + // destroys scalars and empty containers only, so it recurses at most + // one level deep); stops at the first structured child it meets + static bool all_children_leaves(const basic_json& v) noexcept + { + if (v.m_data.m_type == value_t::array) + { + const auto& a = *v.m_data.m_value.array; + for (auto it = a.rbegin(); it != a.rend(); ++it) + { + if (!has_no_children(*it)) + { + return false; + } + } + return true; + } + for (const auto& el : *v.m_data.m_value.object) + { + if (!has_no_children(el.second)) + { + return false; + } + } + return true; + } + // The walk in destroy_container() may take the children of a // container in any order, as long as it picks the same child again // while that container is not modified in between. Arrays and @@ -31168,15 +31909,16 @@ private: } } - // deallocates the (already empty) array/object held by v; this is - // the same allocator-based free the old recursive implementation - // used, just factored out so every level of the walk in destroy() - // can share it + // deallocates the array/object held by v, which is either empty or + // holds only scalars and empty containers (so its destructor + // recurses at most one level deep); this is the same allocator-based + // free the old recursive implementation used, just factored out so + // every level of the walk in destroy() can share it static void free_container(basic_json& v) noexcept { + JSON_ASSERT(has_no_children(v) || all_children_leaves(v)); if (v.m_data.m_type == value_t::array) { - JSON_ASSERT(v.m_data.m_value.array->empty()); AllocatorType alloc; std::allocator_traits::destroy(alloc, v.m_data.m_value.array); std::allocator_traits::deallocate(alloc, v.m_data.m_value.array, 1); @@ -31184,7 +31926,6 @@ private: else { JSON_ASSERT(v.m_data.m_type == value_t::object); - JSON_ASSERT(v.m_data.m_value.object->empty()); AllocatorType alloc; std::allocator_traits::destroy(alloc, v.m_data.m_value.object); std::allocator_traits::deallocate(alloc, v.m_data.m_value.object, 1); @@ -31229,40 +31970,99 @@ public: return; } - // Destroy the tree without recursing per nesting level and - // without any heap allocation: a heap-allocated flattening - // stack (the previous implementation) can itself throw - // bad_alloc, which would escape this noexcept destructor and - // terminate the program (#5135). - // - // Instead, walk down a chain of children (always the one - // walk_child() picks), reversing links as we go: cur is the - // container currently being emptied, and prev is its parent - // (value_t::null when there is none). Each parent's - // walk_child() slot doubles as storage for that parent's own - // parent link while we are below it, so no extra memory is - // needed; the parent is not modified meanwhile, so - // walk_child() finds that same slot again on the way up. We - // only ever remove a child once it is a scalar or an empty - // array/object, which neither allocates nor recurses more - // than one level deep. - // - // This json_value is not itself a basic_json, so the - // top-level container is first moved into a local stand-in - // ("cur"); a default-constructed basic_json has a null - // pointer in its m_value (see data::m_value's initializer), - // so swapping it with *this leaves this union's own pointer - // null, and it is never looked at or freed a second time. + // This json_value is not itself a basic_json, so the top-level + // container is first moved into a local stand-in ("cur"); a + // default-constructed basic_json has a null pointer in its + // m_value (see data::m_value's initializer), so swapping it + // with *this leaves this union's own pointer null, and it is + // never looked at or freed a second time. basic_json cur; cur.m_data.m_type = t; using std::swap; swap(cur.m_data.m_value, *this); + destroy_bounded(cur, 0); + } + + // Destroy the array/object v, whose children are destroyed first. + // The first recursion_depth_limit() levels are destroyed by plain + // recursion, which is the fastest way and cannot throw; anything + // nested deeper is handed to destroy_walk(), which uses no call stack. + static void destroy_bounded(basic_json& v, std::size_t depth) noexcept + { + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + destroy_walk(v); + return; + } + + if (v.m_data.m_type == value_t::array) + { + for (auto& child : *v.m_data.m_value.array) + { + if (child.m_data.m_type == value_t::array || child.m_data.m_type == value_t::object) + { + destroy_bounded(child, depth + 1); + } + } + } + else + { + for (auto& child : *v.m_data.m_value.object) + { + if (child.second.m_data.m_type == value_t::array || child.second.m_data.m_type == value_t::object) + { + destroy_bounded(child.second, depth + 1); + } + } + } + + // only leaves and already destroyed (null) children remain + free_container(v); + } + + // Destroy the tree below top without recursing per nesting level and + // without any heap allocation: a heap-allocated flattening + // stack (the previous implementation) can itself throw + // bad_alloc, which would escape this noexcept destructor and + // terminate the program (#5135). + // + // Instead, walk down a chain of children (always the one + // walk_child() picks), reversing links as we go: cur is the + // container currently being emptied, and prev is its parent + // (value_t::null when there is none). Each parent's + // walk_child() slot doubles as storage for that parent's own + // parent link while we are below it, so no extra memory is + // needed; the parent is not modified meanwhile, so + // walk_child() finds that same slot again on the way up. We + // only ever remove a child once it is a scalar or an empty + // array/object, which neither allocates nor recurses more + // than one level deep. + // + // This json_value is not itself a basic_json, so the + // top-level container is first moved into a local stand-in + // ("cur"); a default-constructed basic_json has a null + // pointer in its m_value (see data::m_value's initializer), + // so swapping it with *this leaves this union's own pointer + // null, and it is never looked at or freed a second time. + static void destroy_walk(basic_json& top) noexcept + { + basic_json cur; + take(cur, top); + basic_json prev; // value_t::null: no parent + // true right after cur was entered: its children have not been + // looked at yet. If none of them has children of its own, cur is + // freed in one go instead of child by child; this is tried once + // per container, so the walk stays linear. + bool fresh = true; + while (true) { - if (has_no_children(cur)) + const bool fresh_now = fresh; + fresh = false; + if (has_no_children(cur) || (fresh_now && all_children_leaves(cur))) { if (prev.m_data.m_type == value_t::null) { @@ -31301,10 +32101,22 @@ public: take(cur_child_ref, prev); take(prev, cur); take(cur, tmp); + fresh = true; } } + // keep the check whether t owns memory inline so destroying a scalar + // is a compare and a branch, and leave the actual work to an + // out-of-line function void destroy(value_t t) + { + if (JSON_HEDLEY_UNLIKELY(detail::owns_memory(t))) + { + destroy_owned(t); + } + } + + JSON_HEDLEY_NEVER_INLINE void destroy_owned(value_t t) { switch (t) { @@ -31472,11 +32284,16 @@ public: return next; } - /// @brief the capacity of the stored array, or unknown_size() - /// Only JSON_DIAGNOSTICS uses the value, to detect a reallocation that - /// would invalidate the parent pointers. Array types that do not have a - /// capacity() member function report unknown_size(), which is treated as - /// "the elements may have moved". + /// @brief the capacity of the stored array or object, or unknown_size() + /// Only JSON_DIAGNOSTICS uses the value, to detect an insertion that may + /// have moved the other elements and so invalidated their parent pointers. + /// unknown_size() means "the elements may have moved". Arrays only grow at + /// the end, so their capacity tells whether they reallocated; arrays + /// without a capacity() member report unknown_size(). For objects, only + /// types known to keep their elements in place (std::map and the other + /// node-based standard containers) report a fixed 0, and only + /// nlohmann::ordered_map, which appends members, reports its capacity; any + /// other object type (a sorted flat map, say) reports unknown_size(). #if JSON_DIAGNOSTICS template < typename A = array_t, detail::enable_if_t < detail::has_capacity::value, int > = 0 > std::size_t array_capacity() const noexcept @@ -31489,57 +32306,63 @@ public: { return detail::unknown_size(); } + + template < typename O = object_t, detail::enable_if_t < detail::has_stable_elements::value, int > = 0 > + static constexpr std::size_t object_capacity() noexcept + { + return 0; + } + + template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && detail::is_nlohmann_ordered_map::value, int > = 0 > + std::size_t object_capacity() const noexcept + { + return m_data.m_value.object->capacity(); + } + + template < typename O = object_t, detail::enable_if_t < !detail::has_stable_elements::value && !detail::is_nlohmann_ordered_map::value, int > = 0 > + static constexpr std::size_t object_capacity() noexcept + { + return detail::unknown_size(); + } + + std::size_t container_capacity() const noexcept + { + return is_array() ? array_capacity() : object_capacity(); + } #else - static constexpr std::size_t array_capacity() noexcept + static constexpr std::size_t container_capacity() noexcept { return detail::unknown_size(); } #endif - /// @brief set the parent of a value that has just been added to an array - /// @param j the added value - /// @param old_capacity the value @ref array_capacity() returned before the + /// @brief set the parent of a value that has just been added to an array or object + /// @param j the added (or looked-up) value + /// @param old_capacity the value @ref container_capacity() returned before the /// insertion - reference set_parent_after_array_insert(reference j, std::size_t old_capacity) + reference set_parent_after_insert(reference j, std::size_t old_capacity) { #if JSON_DIAGNOSTICS // see https://github.com/nlohmann/json/issues/2838 - JSON_ASSERT(type() == value_t::array); if (JSON_HEDLEY_UNLIKELY(old_capacity == detail::unknown_size() - || array_capacity() != old_capacity)) + || container_capacity() != old_capacity)) { - // the capacity has changed, or the array type does not let us tell: - // the elements may have moved, so update all parents + // the capacity has changed, or the container type does not let us + // tell: the elements may have moved, so update all parents set_parents(); return j; } #else static_cast(old_capacity); #endif - return set_parent(j); + return set_parent_at(j); } - reference set_parent(reference j) + /// @brief set the parent of an element accessed via at(); never moves elements + reference set_parent_at(reference j) noexcept { #if JSON_DIAGNOSTICS - // ordered_json uses a vector internally, so pointers could have - // been invalidated; see https://github.com/nlohmann/json/issues/2962 -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - set_parents(); - return j; - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif - j.m_parent = this; -#else - static_cast(j); #endif return j; } @@ -32529,9 +33352,11 @@ public: /// @brief restore the parent pointers after erasing from an object - /// ordered_json keeps its members in a vector, and erasing a member - /// re-constructs every member after it in place, which resets their - /// parent pointers + /// Unless the object type is known to keep its elements in place (see + /// detail::has_stable_elements), erasing a member may move the members + /// after it; ordered_json, for instance, keeps its members in a vector + /// and re-constructs every member after the erased one in place, which + /// resets their parent pointers void set_parents_after_object_erase() { #if JSON_DIAGNOSTICS @@ -32539,7 +33364,7 @@ public: #pragma warning(push ) #pragma warning(disable : 4127) // ignore warning to replace if with if constexpr #endif - if (detail::is_ordered_map::value) + if (!detail::has_stable_elements::value) { set_parents(); } @@ -32574,7 +33399,7 @@ public: /// @brief create an empty value with a given type /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ basic_json(const value_t v) - : m_data(v) + : m_data(stored_kind(v)) { assert_invariant(); } @@ -32655,7 +33480,7 @@ public: { convert_leaf(val); } - JSON_ASSERT(m_data.m_type == val.type()); + JSON_ASSERT(m_data.m_type == stored_kind(val.type())); set_parents(); assert_invariant(); @@ -32703,7 +33528,7 @@ public: // if an object is wanted but impossible, throw an exception if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object)) { - JSON_THROW(type_error::create(detail::exception_id::object_from_non_pairs, "cannot create object from initializer list", nullptr)); + detail::throw_literal(detail::exception_id::object_from_non_pairs, "cannot create object from initializer list", nullptr); } } @@ -32823,7 +33648,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterators_incompatible, "iterators are not compatible", nullptr)); + detail::throw_literal(detail::exception_id::iterators_incompatible, "iterators are not compatible", nullptr); } // copy type from the first iterator @@ -32842,7 +33667,7 @@ public: if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() || !last.m_it.primitive_iterator.is_end())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_out_of_range, "iterators out of range", first.m_object)); + detail::throw_literal(detail::exception_id::iterator_range_out_of_range, "iterators out of range", first.m_object); } break; } @@ -32910,7 +33735,7 @@ public: case value_t::null: case value_t::discarded: default: - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", first.m_object->type_name()), first.m_object)); + throw_iterator_range_of_null(first.m_object); } set_parents(); @@ -32939,18 +33764,16 @@ public: // check of passed value is valid other.assert_invariant(); - if (m_data.m_type == value_t::object || m_data.m_type == value_t::array) + if (JSON_HEDLEY_LIKELY(!detail::owns_memory(m_data.m_type))) { - // copying the container directly would call this constructor again - // for every element, once per nesting level - copy_structured(other); + // null, boolean, numbers, discarded: plain copy of the union + m_data.m_value = other.m_data.m_value; } else { - copy_leaf_value(other, *this); + copy_owning(other); } - set_parents(); assert_invariant(); } @@ -33180,6 +34003,22 @@ public: // value access // ////////////////// + /// out-of-line part of the copy constructor for owning types + JSON_HEDLEY_NEVER_INLINE + void copy_owning(const basic_json& other) + { + if (m_data.m_type == value_t::object || m_data.m_type == value_t::array) + { + // copying the container directly would call this constructor again + // for every element, once per nesting level + copy_structured(other); + } + else + { + copy_leaf_value(other, *this); + } + } + /// get a pointer to the value (object) object_t* get_impl_ptr(object_t* /*unused*/) noexcept { @@ -33240,14 +34079,16 @@ public: return m_data.m_type == value_t::number_integer ? &m_data.m_value.number_integer : nullptr; } - /// get a pointer to the value (unsigned number) - number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + /// get a pointer to the value (unsigned number); takes a placeholder if + /// number_unsigned_t is number_integer_t (the integer overload handles it then) + number_unsigned_t* get_impl_ptr(detail::number_unsigned_overload_type* /*unused*/) noexcept { return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } - /// get a pointer to the value (unsigned number) - constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + /// get a pointer to the value (unsigned number); takes a placeholder if + /// number_unsigned_t is number_integer_t (the integer overload handles it then) + constexpr const number_unsigned_t* get_impl_ptr(const detail::number_unsigned_overload_type* /*unused*/) const noexcept { return is_number_unsigned() ? &m_data.m_value.number_unsigned : nullptr; } @@ -33276,6 +34117,40 @@ public: return is_binary() ? m_data.m_value.binary : nullptr; } + /// out-of-line throw helpers (message construction kept out of the hot paths) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_iterator_range_of_null(const basic_json* obj) + { + JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_of_null, detail::concat("cannot construct with iterators from ", obj->type_name()), obj)); + } + + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_incompatible_reference_type(const basic_json& obj) + { + JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); + } + + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_array_index_exceeds_size_type(size_type idx) + { + JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); + } + + /// @throw out_of_range.401 + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_array_index_out_of_range(size_type idx, const basic_json& j) + { + JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + } + + /// @throw out_of_range.403 + template + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_key_not_found(const KeyType& key, const basic_json& j) + { + JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key, "' not found"), &j)); + } + /*! @brief helper function to implement get_ref() @@ -33298,7 +34173,7 @@ public: return *ptr; } - JSON_THROW(type_error::create(detail::exception_id::incompatible_reference_type, detail::concat("incompatible ReferenceType for get_ref, actual type is ", obj.type_name()), &obj)); + throw_incompatible_reference_type(obj); } public: @@ -33746,7 +34621,7 @@ public: // std::map or ordered_map) never moves from its argument, so key is still // valid here regardless of whether KeyType was deduced as an rvalue reference // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved) - JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", key_for_message(key), "' not found"), &j)); + throw_key_not_found(key_for_message(key), j); } return it->second; } @@ -33783,7 +34658,7 @@ public: if (JSON_HEDLEY_UNLIKELY(idx >= j.m_data.m_value.array->size())) { - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &j)); + throw_array_index_out_of_range(idx, j); } return (*j.m_data.m_value.array)[idx]; @@ -33838,7 +34713,7 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(size_type idx) { - return set_parent(array_at(*this, idx)); + return set_parent_at(array_at(*this, idx)); } /// @brief access specified array element with bounds checking @@ -33852,7 +34727,7 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/at/ reference at(const typename object_t::key_type& key) { - return set_parent(object_at(*this, key)); + return set_parent_at(object_at(*this, key)); } /// @brief access specified object element with bounds checking @@ -33861,7 +34736,7 @@ public: detail::is_usable_as_basic_json_key_type::value, int> = 0> reference at(KeyType && key) { - return set_parent(object_at(*this, std::forward(key))); + return set_parent_at(object_at(*this, std::forward(key))); } /// @brief access specified object element with bounds checking @@ -33901,18 +34776,18 @@ public: // resize() rejects other indices that are too large to represent if (JSON_HEDLEY_UNLIKELY(idx == (std::numeric_limits::max)())) { - JSON_THROW(std::length_error(detail::concat("array index ", std::to_string(idx), " exceeds size_type"))); + throw_array_index_exceeds_size_type(idx); } #if JSON_DIAGNOSTICS // remember array size & capacity before resizing const auto old_size = m_data.m_value.array->size(); - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); #endif m_data.m_value.array->resize(idx + 1); #if JSON_DIAGNOSTICS if (JSON_HEDLEY_UNLIKELY(old_capacity == detail::unknown_size() - || array_capacity() != old_capacity)) + || container_capacity() != old_capacity)) { // capacity has changed: update all parents set_parents(); @@ -33959,8 +34834,9 @@ public: // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { + const auto old_capacity = container_capacity(); auto result = m_data.m_value.object->emplace(std::move(key), nullptr); - return set_parent(result.first->second); + return set_parent_after_insert(result.first->second, old_capacity); } throw_subscript_wrong_type("string"); @@ -34010,8 +34886,9 @@ public: // operator[] only works for objects if (JSON_HEDLEY_LIKELY(is_object())) { + const auto old_capacity = container_capacity(); auto result = m_data.m_value.object->emplace(lookup_key(std::forward(key)), nullptr); - return set_parent(result.first->second); + return set_parent_after_insert(result.first->second, old_capacity); } throw_subscript_wrong_type("string"); @@ -34237,7 +35114,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(this != pos.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } IteratorType result = end(); @@ -34253,7 +35130,7 @@ public: { if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_out_of_range, "iterator out of range", this)); + detail::throw_literal(detail::exception_id::iterator_out_of_range, "iterator out of range", this); } m_data.m_value.destroy(m_data.m_type); @@ -34295,7 +35172,7 @@ public: // make sure the iterator fits the current value if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_from_other_value, "iterators do not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_range_from_other_value, "iterators do not fit current value", this); } IteratorType result = end(); @@ -34312,7 +35189,7 @@ public: if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() || !last.m_it.primitive_iterator.is_end())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_range_out_of_range, "iterators out of range", this)); + detail::throw_literal(detail::exception_id::iterator_range_out_of_range, "iterators out of range", this); } m_data.m_value.destroy(m_data.m_type); @@ -34411,7 +35288,7 @@ public: { if (JSON_HEDLEY_UNLIKELY(idx >= size())) { - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), this)); + throw_array_index_out_of_range(idx, *this); } m_data.m_value.array->erase(m_data.m_value.array->begin() + static_cast(idx)); @@ -34917,9 +35794,9 @@ public: } // add the element to the array (move semantics) - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->push_back(std::move(val)); - set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + set_parent_after_insert(m_data.m_value.array->back(), old_capacity); // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor } @@ -34948,9 +35825,9 @@ public: } // add the element to the array - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->push_back(val); - set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + set_parent_after_insert(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an array @@ -34978,8 +35855,9 @@ public: } // add the element to the object + const auto old_capacity = container_capacity(); auto res = m_data.m_value.object->insert(val); - set_parent(res.first->second); + set_parent_after_insert(res.first->second, old_capacity); } /// @brief add an object to an object @@ -35032,9 +35910,9 @@ public: } // add the element to the array (perfect forwarding) - const auto old_capacity = array_capacity(); + const auto old_capacity = container_capacity(); m_data.m_value.array->emplace_back(std::forward(args)...); - return set_parent_after_array_insert(m_data.m_value.array->back(), old_capacity); + return set_parent_after_insert(m_data.m_value.array->back(), old_capacity); } /// @brief add an object to an object if key does not exist @@ -35055,8 +35933,9 @@ public: } // add the element to the array (perfect forwarding) + const auto old_capacity = container_capacity(); auto res = m_data.m_value.object->emplace(std::forward(args)...); - set_parent(res.first->second); + set_parent_after_insert(res.first->second, old_capacity); // create a result iterator and set iterator to the result of emplace auto it = begin(); @@ -35097,7 +35976,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // insert to array and return iterator @@ -35117,7 +35996,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // moving into a local first keeps this safe even if val aliases @@ -35139,7 +36018,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // insert to array and return iterator @@ -35162,24 +36041,24 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } if (JSON_HEDLEY_UNLIKELY(first.m_object == this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_into_itself, "passed iterators may not belong to container", this)); + detail::throw_literal(detail::exception_id::insert_range_into_itself, "passed iterators may not belong to container", this); } // passed iterators must belong to arrays if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_array())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterators first and last must point to arrays", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to arrays", this); } // insert to array and return iterator @@ -35199,7 +36078,7 @@ public: // check if iterator pos fits to this JSON value if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterator does not fit current value", this); } // copy the values first: ilist may refer to elements of this array @@ -35227,13 +36106,13 @@ public: // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } // passed iterators must belong to objects if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) { - JSON_THROW(invalid_iterator::create(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this)); + detail::throw_literal(detail::exception_id::iterator_from_other_value, "iterators first and last must point to objects", this); } m_data.m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); @@ -35268,7 +36147,7 @@ public: // check if range iterators belong to the same JSON object if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) { - JSON_THROW(invalid_iterator::create(detail::exception_id::insert_range_incompatible, "iterators do not fit", this)); + detail::throw_literal(detail::exception_id::insert_range_incompatible, "iterators do not fit", this); } // passed iterators must belong to objects @@ -35355,9 +36234,9 @@ public: continue; } } - // set_parent() also repairs the other members, which ordered_json - // relocates when adding a key makes its vector grow - set_parent(m_data.m_value.object->operator[](it.key()) = std::move(it.value())); + // operator[] repairs the other members if the object type moves + // them; assigning through it keeps this O(1) per key + operator[](it.key()) = std::move(it.value()); } } @@ -35407,9 +36286,8 @@ public: continue; } } - // set_parent() also repairs the other members, which ordered_json - // relocates when adding a key makes its vector grow - target->set_parent(target->m_data.m_value.object->operator[](first.key()) = std::move(first.value())); + // operator[] repairs the other members if the object type moves them + target->operator[](first.key()) = std::move(first.value()); ++first; } } @@ -35551,7 +36429,22 @@ public: /// @name lexicographical comparison operators /// @{ - // note parentheses around operands are necessary; see + // the same-type cases for non-owning scalars, shared by JSON_IMPLEMENT_OPERATOR + // and JSON_IMPLEMENT_SCALAR_FAST +#define JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + case value_t::null: \ + return (null_result); \ + case value_t::boolean: \ + return lhs.m_data.m_value.boolean op rhs.m_data.m_value.boolean; \ + case value_t::number_integer: \ + return lhs.m_data.m_value.number_integer op rhs.m_data.m_value.number_integer; \ + case value_t::number_unsigned: \ + return lhs.m_data.m_value.number_unsigned op rhs.m_data.m_value.number_unsigned; \ + case value_t::number_float: \ + return lhs.m_data.m_value.number_float op rhs.m_data.m_value.number_float; + + // note parentheses around the container operands are necessary, because + // GCC otherwise parses `array <` as std::array; see // https://github.com/nlohmann/json/issues/1530 // Mixed signed/unsigned integer comparisons check whether the signed value // is negative before casting. If it is, the comparison is performed with @@ -35587,24 +36480,11 @@ public: return (*lhs.m_data.m_value.object) op (*rhs.m_data.m_value.object); \ } \ \ - case value_t::null: \ - return (null_result); \ - \ + JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + \ case value_t::string: \ return (*lhs.m_data.m_value.string) op (*rhs.m_data.m_value.string); \ \ - case value_t::boolean: \ - return (lhs.m_data.m_value.boolean) op (rhs.m_data.m_value.boolean); \ - \ - case value_t::number_integer: \ - return (lhs.m_data.m_value.number_integer) op (rhs.m_data.m_value.number_integer); \ - \ - case value_t::number_unsigned: \ - return (lhs.m_data.m_value.number_unsigned) op (rhs.m_data.m_value.number_unsigned); \ - \ - case value_t::number_float: \ - return (lhs.m_data.m_value.number_float) op (rhs.m_data.m_value.number_float); \ - \ case value_t::binary: \ return (*lhs.m_data.m_value.binary) op (*rhs.m_data.m_value.binary); \ \ @@ -35648,6 +36528,24 @@ public: \ return (default_result); + // inline fast path for two values of the same non-owning scalar type; anything + // else falls through to the out-of-line general comparison +#define JSON_IMPLEMENT_SCALAR_FAST(op, null_result) \ + if (lhs.m_data.m_type == rhs.m_data.m_type) \ + { \ + switch (lhs.m_data.m_type) \ + { \ + JSON_IMPLEMENT_SCALAR_CASES(op, null_result) \ + case value_t::object: \ + case value_t::array: \ + case value_t::string: \ + case value_t::binary: \ + case value_t::discarded: \ + default: \ + break; \ + } \ + } + JSON_PRIVATE_UNLESS_TESTED: // returns true if: // - any operand is NaN and the other operand is of number type @@ -35675,6 +36573,173 @@ public: return compares_unordered(*this, rhs, inverse); } + /// @brief whether objects and arrays are the standard containers, whose + /// elements equal_bounded() may therefore compare itself + using standard_object_t = std::map>>; + using standard_array_t = std::vector>; + using equal_bounded_containers = detail::conjunction, std::is_same>; + + // out-of-line general comparisons; the operators below handle same-type + // scalars inline and fall through to these + JSON_HEDLEY_NEVER_INLINE + static bool equal_general(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + JSON_IMPLEMENT_OPERATOR( ==, true, false, false, + compare_iteratively(lhs, rhs, false) == compare_result::equal, true) +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + } + + JSON_HEDLEY_NEVER_INLINE + static bool less_general(const_reference lhs, const_reference rhs) noexcept + { + // default_result is used if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), + compare_iteratively(lhs, rhs, true) == compare_result::less, false) + } + +#if JSON_HAS_THREE_WAY_COMPARISON + JSON_HEDLEY_NEVER_INLINE + static std::partial_ordering three_way_general(const_reference lhs, const_reference rhs) noexcept // *NOPAD* + { + // default_result is used if we cannot compare values. In that case, + // we compare types. + JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* + std::partial_ordering::equivalent, + std::partial_ordering::unordered, + lhs_type <=> rhs_type, // *NOPAD* + to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) + } +#endif + + /// @brief compare two leaves of the same type inside equal_structured() + /// @return 1 if they are equal, 0 if they are not, and -1 if the values are + /// of different types or are not handled here, which this leaves to + /// equal_structured() + /// @note Unlike JSON_IMPLEMENT_SCALAR_FAST, this also compares strings: + /// they are the most common leaves in a document, so comparing them + /// inline pays off in the container walk, while the operators keep + /// their inlined code at each call site small. Binary values are + /// rare and left to equal_general(). + JSON_HEDLEY_ALWAYS_INLINE + static int equal_leaf(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + if (lhs.m_data.m_type == rhs.m_data.m_type) + { + switch (lhs.m_data.m_type) + { + case value_t::null: + return 1; + case value_t::boolean: + return lhs.m_data.m_value.boolean == rhs.m_data.m_value.boolean; + case value_t::number_integer: + return lhs.m_data.m_value.number_integer == rhs.m_data.m_value.number_integer; + case value_t::number_unsigned: + return lhs.m_data.m_value.number_unsigned == rhs.m_data.m_value.number_unsigned; + case value_t::number_float: + return lhs.m_data.m_value.number_float == rhs.m_data.m_value.number_float; + case value_t::string: + return *lhs.m_data.m_value.string == *rhs.m_data.m_value.string; + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + } +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + return -1; + } + + /// @brief compare for equality, walking objects and arrays here instead of + /// in their containers so that the depth is a parameter and needs no + /// thread-local count; only used for the standard containers + static bool equal_bounded(const_reference lhs, const_reference rhs, std::size_t depth) noexcept + { + const int leaf = equal_leaf(lhs, rhs); + return leaf < 0 ? equal_structured(lhs, rhs, depth) : leaf != 0; + } + + static bool equal_structured(const_reference lhs, const_reference rhs, std::size_t depth) noexcept + { + const auto type = lhs.m_data.m_type; + if (type != rhs.m_data.m_type || (type != value_t::array && type != value_t::object)) + { + return equal_general(lhs, rhs); + } + + if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) + { + return compare_iteratively(lhs, rhs, false) == compare_result::equal; + } + + if (type == value_t::array) + { + const auto& a = *lhs.m_data.m_value.array; + const auto& b = *rhs.m_data.m_value.array; + if (a.size() != b.size()) + { + return false; + } + auto it_b = b.cbegin(); + for (auto it_a = a.cbegin(); it_a != a.cend(); ++it_a, ++it_b) + { + const int leaf = equal_leaf(*it_a, *it_b); + if (leaf < 0 ? !equal_structured(*it_a, *it_b, depth + 1) : leaf == 0) + { + return false; + } + } + return true; + } + + const auto& a = *lhs.m_data.m_value.object; + const auto& b = *rhs.m_data.m_value.object; + if (a.size() != b.size()) + { + return false; + } + auto it_b = b.cbegin(); + for (auto it_a = a.cbegin(); it_a != a.cend(); ++it_a, ++it_b) + { + if (!(it_a->first == it_b->first)) + { + return false; + } + const int leaf = equal_leaf(it_a->second, it_b->second); + if (leaf < 0 ? !equal_structured(it_a->second, it_b->second, depth + 1) : leaf == 0) + { + return false; + } + } + return true; + } + + static bool equal_dispatch(const_reference lhs, const_reference rhs, std::true_type /*standard containers*/) noexcept + { + return equal_bounded(lhs, rhs, 0); + } + + static bool equal_dispatch(const_reference lhs, const_reference rhs, std::false_type /*other containers*/) noexcept + { + return equal_general(lhs, rhs); + } + public: #if JSON_HAS_THREE_WAY_COMPARISON /// @brief comparison: equal @@ -35686,11 +36751,11 @@ public: JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif const_reference lhs = *this; - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, - compare_iteratively(lhs, rhs, false) == compare_result::equal, true) + JSON_IMPLEMENT_SCALAR_FAST( ==, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP #endif + return equal_dispatch(lhs, rhs, equal_bounded_containers {}); } /// @brief comparison: equal @@ -35707,13 +36772,8 @@ public: std::partial_ordering operator<=>(const_reference rhs) const noexcept // *NOPAD* { const_reference lhs = *this; - // default_result is used if we cannot compare values. In that case, - // we compare types. - JSON_IMPLEMENT_OPERATOR(<=>, // *NOPAD* - std::partial_ordering::equivalent, - std::partial_ordering::unordered, - lhs_type <=> rhs_type, // *NOPAD* - to_partial_ordering(compare_iteratively(lhs, rhs, false)), true) + JSON_IMPLEMENT_SCALAR_FAST(<=>, std::partial_ordering::equivalent) // *NOPAD* + return three_way_general(lhs, rhs); } /// @brief comparison: 3-way @@ -35801,11 +36861,11 @@ public: JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") #endif - JSON_IMPLEMENT_OPERATOR( ==, true, false, false, - compare_iteratively(lhs, rhs, false) == compare_result::equal, true) + JSON_IMPLEMENT_SCALAR_FAST( ==, true) #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_POP #endif + return equal_dispatch(lhs, rhs, equal_bounded_containers {}); } // The friend comparisons with a scalar name the JSON type via decltype of @@ -35860,11 +36920,8 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ friend bool operator<(const_reference lhs, const_reference rhs) noexcept { - // default_result is used if we cannot compare values. In that case, - // we compare types. Note we have to call the operator explicitly, - // because MSVC has problems otherwise. - JSON_IMPLEMENT_OPERATOR( <, false, false, operator<(lhs_type, rhs_type), - compare_iteratively(lhs, rhs, true) == compare_result::less, false) + JSON_IMPLEMENT_SCALAR_FAST( <, false) + return less_general(lhs, rhs); } /// @brief comparison: less than @@ -35975,6 +37032,8 @@ public: #endif #undef JSON_IMPLEMENT_OPERATOR +#undef JSON_IMPLEMENT_SCALAR_FAST +#undef JSON_IMPLEMENT_SCALAR_CASES /// @} @@ -36955,24 +38014,18 @@ public: }; // wrapper for "add" operation; add value at ptr - const auto operation_add = [&result](json_pointer & ptr, const basic_json & val) + const auto operation_add = [&result](json_pointer & ptr, basic_json val) { // adding to the root of the target document means replacing it if (ptr.empty()) { - result = val; + result = std::move(val); return; } - // make sure the top element of the pointer exists - json_pointer const top_pointer = ptr.top(); - if (top_pointer != ptr) - { - result.at(top_pointer); - } - - // get reference to the parent of the JSON pointer ptr - const auto last_path = ptr.back(); + // get reference to the parent of the JSON pointer ptr; a missing + // parent (or one of its ancestors) is reported by at() + auto last_path = std::move(ptr.reference_tokens.back()); ptr.pop_back(); // parent must exist when performing patch add per RFC6902 specs basic_json& parent = result.at(ptr); @@ -36983,7 +38036,7 @@ public: case value_t::object: { // use operator[] to add value - parent[last_path] = val; + parent[std::move(last_path)] = std::move(val); break; } @@ -36992,7 +38045,7 @@ public: if (last_path == "-") { // special case: append to back - parent.push_back(val); + parent.push_back(std::move(val)); } else { @@ -37000,11 +38053,11 @@ public: if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) { // avoid undefined behavior - JSON_THROW(out_of_range::create(detail::exception_id::array_index_out_of_range, detail::concat("array index ", std::to_string(idx), " is out of range"), &parent)); + throw_array_index_out_of_range(idx, parent); } // default case: insert add offset - parent.insert(parent.begin() + static_cast(idx), val); + parent.insert(parent.begin() + static_cast(idx), std::move(val)); } break; } @@ -37019,7 +38072,7 @@ public: case value_t::binary: case value_t::discarded: default: - JSON_THROW(out_of_range::create(detail::exception_id::patch_add_parent_not_container, detail::concat("cannot add value: the JSON Patch 'add' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + detail::throw_patch_add_parent(parent); } }; @@ -37027,7 +38080,8 @@ public: const auto operation_remove = [this, & result](json_pointer & ptr) { // get reference to the parent of the JSON pointer ptr - const auto last_path = ptr.back(); + static_cast(ptr.back()); // throws out_of_range.405 if ptr is the root + const auto last_path = std::move(ptr.reference_tokens.back()); ptr.pop_back(); basic_json& parent = result.at(ptr); @@ -37042,7 +38096,7 @@ public: } else { - JSON_THROW(out_of_range::create(detail::exception_id::key_not_found, detail::concat("key '", last_path, "' not found"), this)); + throw_key_not_found(last_path, *this); } } else if (parent.is_array()) @@ -37054,7 +38108,7 @@ public: { // the parent of a "remove" target must be an object or array // (see #5396) - JSON_THROW(out_of_range::create(detail::exception_id::patch_remove_parent_not_container, detail::concat("cannot remove value: the JSON Patch 'remove' target's parent is of type ", parent.type_name(), ", but must be an object or array"), &parent)); + detail::throw_patch_remove_parent(parent); } }; @@ -37085,7 +38139,7 @@ public: // type check: top level value must be an array if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array())) { - JSON_THROW(parse_error::create(detail::exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &json_patch)); + detail::throw_patch_not_an_array(json_patch); } // iterate and apply the operations @@ -37099,21 +38153,16 @@ public: // find value auto it = val.m_data.m_value.object->find(member); - // context-sensitive error message - const auto error_msg = (op == "op") ? "operation" : detail::concat("operation '", op, '\''); // NOLINT(bugprone-unused-local-non-trivial-variable) - // check if the desired value is present if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end())) { - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat(error_msg, " must have member '", member, "'"), &val)); + detail::throw_patch_missing_member(op, member, false, val); } // check if the result is of type string if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) { - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat(error_msg, " must have string member '", member, "'"), &val)); + detail::throw_patch_missing_member(op, member, true, val); } // no error: return value @@ -37123,12 +38172,12 @@ public: // type check: every element of the array must be an object if (JSON_HEDLEY_UNLIKELY(!val.is_object())) { - JSON_THROW(parse_error::create(detail::exception_id::patch_not_an_array, 0, "JSON patch must be an array of objects", &val)); + detail::throw_patch_not_an_array(val); } // collect mandatory members - const auto op = get_value("op", "op", true).template get(); - const auto path = get_value(op, "path", true).template get(); + const string_t& op = *get_value("op", "op", true).m_data.m_value.string; + const string_t& path = *get_value(op, "path", true).m_data.m_value.string; json_pointer ptr(path); switch (get_op(op)) @@ -37154,38 +38203,37 @@ public: case patch_operations::move: { - const auto from_path = get_value("move", "from", true).template get(); + const string_t& from_path = *get_value("move", "from", true).m_data.m_value.string; json_pointer from_ptr(from_path); if (JSON_HEDLEY_UNLIKELY(is_proper_prefix(from_ptr, ptr))) { - JSON_THROW(out_of_range::create(detail::exception_id::patch_move_into_child, detail::concat("cannot move value: 'from' path '", from_path, "' is a proper prefix of 'path' '", path, "'"), &result)); + detail::throw_patch_move_into_child(from_path, path, result); } // the "from" location must exist - use at() - basic_json const v = result.at(from_ptr); + // a root "from" is rejected by operation_remove below, which must leave result intact + basic_json v = from_ptr.empty() ? basic_json(result) : std::move(result.at(from_ptr)); // The move operation is functionally identical to a // "remove" operation on the "from" location, followed // immediately by an "add" operation at the target // location with the value that was just removed. operation_remove(from_ptr); - operation_add(ptr, v); + operation_add(ptr, std::move(v)); break; } case patch_operations::copy: { - const auto from_path = get_value("copy", "from", true).template get(); + const string_t& from_path = *get_value("copy", "from", true).m_data.m_value.string; const json_pointer from_ptr(from_path); - // the "from" location must exist - use at() - basic_json const v = result.at(from_ptr); - // The copy is functionally identical to an "add" // operation at the target location using the value - // specified in the "from" member. - operation_add(ptr, v); + // specified in the "from" member; the "from" location + // must exist - use at() + operation_add(ptr, result.at(from_ptr)); break; } @@ -37206,7 +38254,7 @@ public: // throw an exception if the test fails if (JSON_HEDLEY_UNLIKELY(!success)) { - JSON_THROW(other_error::create(detail::exception_id::patch_test_failed, detail::concat("unsuccessful: ", val.dump()), &val)); + detail::throw_patch_test_failed(val); } break; @@ -37217,7 +38265,7 @@ public: { // op must be "add", "remove", "replace", "move", "copy", or // "test" - JSON_THROW(parse_error::create(detail::exception_id::patch_invalid_operation, 0, detail::concat("operation value '", op, "' is invalid"), &val)); + detail::throw_patch_invalid_operation(op, val); } } } @@ -37271,12 +38319,11 @@ public: std::size_t index = 0; /// objects: the next member of source to look at const_iterator member{}; // NOLINT(readability-redundant-member-init) - /// objects: the keys common to both, in source's order - std::vector common_keys{}; // NOLINT(readability-redundant-member-init) - /// objects: the next entry of common_keys - std::size_t next_common = 0; - /// objects: the "add" operations for keys only target has - basic_json added_ops{}; // NOLINT(readability-redundant-member-init) + /// objects: for each member of source (in order), its value in + /// target, or nullptr if target has no such key; `index` is the next + std::vector target_values{}; // NOLINT(readability-redundant-member-init) + /// objects: the members only target has, in target's order + std::vector added{}; // NOLINT(readability-redundant-member-init) }; // The operations of a diff are built by the functions below rather than @@ -37332,111 +38379,113 @@ public: } } + /// @brief appends the "add" operations for the members @a added to @a result + static void diff_added(basic_json& result, const std::vector& added, const string_t& path) + { + for (const auto& it : added) + { + diff_add(result, detail::concat(path, '/', detail::escape(it.key())), it.value()); + } + } + /*! @brief compare the keys of objects @a source and @a target If object_t does not keep its members in insertion order, or if the keys both objects have are in the same order in both, and the keys only - @a target has come after them, stores the keys common to both in - source's order in @a common_keys, stores the "add" operations for the keys - only @a target has in @a added_ops, and returns true: the caller then diffs - the objects member by member. Otherwise, appends operations that remove - every member of @a source and add every member of @a target to @a result, - and returns false. + @a target has come after them, stores in @a target_values, for each member + of source in order, its value in target (nullptr for a key target does not + have), stores the members only target has in @a added, and returns true: + the caller then diffs the objects member by member. Otherwise, appends + operations that remove every member of @a source and add every member of + @a target to @a result, and returns false. */ static bool diff_object_keys(basic_json& result, const basic_json& source, const basic_json& target, - const string_t& path, std::vector& common_keys, - basic_json& added_ops) + const string_t& path, std::vector& target_values, + std::vector& added) { - // first pass: record, for every source key, whether it is - // common to both objects (in source's iteration order) or - // was deleted (i.e., in source but not in target) -- this is - // a by-product of the target.find() call already needed to - // tell the two cases apart, so it adds no extra lookups. The - // "remove" ops themselves are emitted later, interleaved - // with the per-key diffs in the caller's fast path, to match - // source's original iteration order (as the original, - // pre-reordering-aware implementation did) instead of - // grouping all removes before all per-key diffs. - std::vector common_keys_source_order; + // first pass: one lookup per source key + target_values.reserve(source.size()); + std::size_t common = 0; for (auto it = source.cbegin(); it != source.cend(); ++it) { - if (target.find(it.key()) != target.end()) + const auto found = target.find(it.key()); + if (found != target.cend()) { - common_keys_source_order.push_back(it.key()); - } - } - - // second pass: find keys that were added (i.e., in target but - // not in source), and record the keys common to both, in - // target's iteration order -- again a by-product of the - // source.find() call already needed to detect added keys. At - // the same time, determine whether every added key comes - // after every common key in target's order (a precondition - // for the fast path, which only ever appends new keys - // at the very end). Both are only needed for an object_t that - // keeps its members in insertion order, such as the one - // backing `ordered_json`; for any other object_t, the fast - // path is always taken and they are not computed. - // The patch ops for keys that were added (i.e., in target but not - // in source) are built here so the fast path can reuse - // them without a second source.find() per target key. Only - // used by the fast path -- the slow (reordering) path - // rebuilds "add" ops for every key itself. - std::vector common_keys_target_order; - bool new_keys_form_suffix = true; - bool seen_new_key = false; - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - if (source.find(it.key()) == source.end()) - { - seen_new_key = true; - diff_add(added_ops, detail::concat(path, '/', detail::escape(it.key())), it.value()); + target_values.push_back(&found.value()); + ++common; } else { -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning(push ) -#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr -#endif - if (detail::is_ordered_map::value) - { - common_keys_target_order.push_back(it.key()); - if (seen_new_key) - { - new_keys_form_suffix = false; - } - } -#ifdef JSON_HEDLEY_MSVC_VERSION -#pragma warning( pop ) -#endif + target_values.push_back(nullptr); } } - // Only an object type that keeps its members in insertion - // order, such as nlohmann::ordered_map, can need reordering: - // patch() appends a new member at the end of such an object. - // Any other object type places its members itself - std::map - // in key order, a hash map in an order its operator== ignores - - // so a member-by-member diff always reproduces target there. + // second pass: find the keys only target has. An object type that + // keeps its members in insertion order, such as the one backing + // ordered_json, also needs the common keys to be in the same order + // in both objects, and the new keys to come after them, as patch() + // appends a new member at the end of such an object. Any other + // object type places its members itself, so a member-by-member diff + // always reproduces target there. #ifdef JSON_HEDLEY_MSVC_VERSION #pragma warning(push ) #pragma warning(disable : 4127) // ignore warning to replace if with if constexpr #endif - if (!detail::is_ordered_map::value - || (common_keys_source_order == common_keys_target_order && new_keys_form_suffix)) + bool fast_path = true; + if (detail::is_ordered_map::value) { - // fast path: order of common keys already matches (or the - // object_t's iteration order does not depend on - // insertion history), so a plain per-key diff is correct - // and minimal, as before - common_keys = std::move(common_keys_source_order); - return true; + // walk the common keys of source in lockstep with target: a key + // of target that is not the next one is either new or out of order + auto src = source.cbegin(); + std::size_t next = 0; + bool seen_new = false; + for (auto it = target.cbegin(); it != target.cend() && fast_path; ++it) + { + while (next < target_values.size() && target_values[next] == nullptr) + { + ++src; + ++next; + } + + if (next < target_values.size() && src.key() == it.key()) + { + // a common key after a new one: new keys are no suffix + fast_path = !seen_new; + ++src; + ++next; + } + else if (source.find(it.key()) == source.cend()) + { + seen_new = true; + added.push_back(it); + } + else + { + // a common key out of order + fast_path = false; + } + } + } + else if (target.size() != common) + { + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.cend()) + { + added.push_back(it); + } + } } #ifdef JSON_HEDLEY_MSVC_VERSION #pragma warning( pop ) #endif + if (fast_path) + { + return true; + } + // slow path: the common keys are in a different relative // order in source and target (only possible for a // reorderable object_t like ordered_map). Building a @@ -37486,6 +38535,14 @@ public: return; } + diff_unequal(result, source, target, path, depth); + } + + /// @brief @ref diff_recursively for values known to be unequal; callers + /// compare first so they build no path for equal ones + static void diff_unequal(basic_json& result, const basic_json& source, const basic_json& target, + const string_t& path, const std::size_t depth) + { if (JSON_HEDLEY_UNLIKELY(depth >= detail::recursion_depth_limit())) { diff_iteratively(result, source, target, path); @@ -37508,7 +38565,10 @@ public: while (i < source.size() && i < target.size()) { // recursive call to compare array values at index i - diff_recursively(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); + if (!(source[i] == target[i])) + { + diff_unequal(result, source[i], target[i], detail::concat(path, '/', detail::to_string(i)), depth + 1); + } ++i; } @@ -37520,25 +38580,24 @@ public: case value_t::object: { - std::vector common_keys; - basic_json added_ops(value_t::array); - if (diff_object_keys(result, source, target, path, common_keys, added_ops)) + std::vector target_values; + std::vector added; + if (diff_object_keys(result, source, target, path, target_values, added)) { - // fast path: common_keys is, by construction, the - // subsequence of source's keys that are common to both - // objects, in source's iteration order -- so it can be - // walked in lockstep with `source` using a cheap key - // comparison instead of another lookup. Deleted keys - // (those source keys not in common_keys) are interleaved - // here too, in source's original order, to match the + // fast path: target_values holds, for every member of + // source in order, its value in target -- so no further + // lookup is needed. Deleted keys are interleaved here + // too, in source's original order, to match the // historical (pre-reordering-aware) output order. - auto common_it = common_keys.cbegin(); - for (auto it = source.cbegin(); it != source.cend(); ++it) + std::size_t i = 0; + for (auto it = source.cbegin(); it != source.cend(); ++it, ++i) { - if (common_it != common_keys.cend() && it.key() == *common_it) + if (target_values[i] != nullptr) { - diff_recursively(result, it.value(), target[it.key()], detail::concat(path, '/', detail::escape(it.key())), depth + 1); - ++common_it; + if (!(it.value() == *target_values[i])) + { + diff_unequal(result, it.value(), *target_values[i], detail::concat(path, '/', detail::escape(it.key())), depth + 1); + } } else { @@ -37547,10 +38606,8 @@ public: } } - // append the "add" ops for brand-new keys collected by - // diff_object_keys -- no second source.find() per target - // key needed - result.insert(result.end(), added_ops.begin(), added_ops.end()); + // append the "add" operations for brand-new keys + diff_added(result, added, path); } break; } @@ -37623,17 +38680,17 @@ public: case value_t::object: { - std::vector common_keys; - basic_json added_ops(value_t::array); - if (diff_object_keys(result, s, t, current_path, common_keys, added_ops)) + std::vector target_values; + std::vector added; + if (diff_object_keys(result, s, t, current_path, target_values, added)) { // fast path: the frame walks source in lockstep with // common_keys, as diff_recursively does, and appends // added_ops once all members are done stack.emplace_back(&s, &t, current_path.size()); stack.back().member = s.cbegin(); - stack.back().common_keys = std::move(common_keys); - stack.back().added_ops = std::move(added_ops); + stack.back().target_values = std::move(target_values); + stack.back().added = std::move(added); } return; } @@ -37695,11 +38752,10 @@ public: if (it != s->cend()) { ++stack.back().member; - const std::size_t next_common = stack.back().next_common; - if (next_common < stack.back().common_keys.size() && it.key() == stack.back().common_keys[next_common]) + const basic_json* const target_value_ptr = stack.back().target_values[stack.back().index++]; + if (target_value_ptr != nullptr) { - ++stack.back().next_common; - const basic_json& target_value = (*t)[it.key()]; + const basic_json& target_value = *target_value_ptr; detail::concat_into(current_path, '/', detail::escape(it.key())); enter(it.value(), target_value); if (stack.size() == depth) @@ -37717,7 +38773,7 @@ public: // append the "add" ops for brand-new keys collected when the // object was entered - result.insert(result.end(), stack.back().added_ops.begin(), stack.back().added_ops.end()); + diff_added(result, stack.back().added, current_path); } // this array or object is done: continue with the one it is in diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index b0511accc..35ee192bc 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -8,10 +8,13 @@ #include #include +#include #include +#include #include -#include #include +#include +#include #include #include @@ -682,4 +685,27 @@ static void FromCborChunkedString(benchmark::State& state, const std::size_t chu BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000); +// conversion of a JSON array into standard containers +template +static void GetArray(benchmark::State& state) +{ + json j = json::array(); + for (int i = 0; i < 1000; ++i) + { + j.push_back(i); + } + + for (auto _ : state) + { + auto c = j.get(); + benchmark::DoNotOptimize(c); + } + state.SetItemsProcessed(state.iterations() * 1000); +} + +BENCHMARK_TEMPLATE(GetArray, std::vector); +BENCHMARK_TEMPLATE(GetArray, std::deque); +BENCHMARK_TEMPLATE(GetArray, std::set); +BENCHMARK_TEMPLATE(GetArray, std::array); + BENCHMARK_MAIN(); diff --git a/tests/fuzzing.md b/tests/fuzzing.md index 329cdbf58..8a1a169ea 100644 --- a/tests/fuzzing.md +++ b/tests/fuzzing.md @@ -9,7 +9,16 @@ Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once once with exceptions. Both calls must agree. Where parsing with exceptions fails, the call without exceptions must return a discarded value (or throw the same kind of non-parse error), and it must never throw a `parse_error`. Where parsing succeeds, both calls must return the same value. The drivers then serialize the value, parse the result back, -and check that nothing was lost. The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`. +and check that nothing was lost. + +Finally, each driver reads the same input through the other input adapters: a `std::istringstream`, a non-contiguous +iterator range (`std::list`), and, on POSIX systems, a `FILE*` from `fmemopen`. The pointer range takes the +fast paths for contiguous input (bulk string scanning, multi-byte loads, lazy position tracking), while the other +adapters read byte by byte. Every adapter must give the same value, or the same exception with the same id, message, +and byte position. The JSON driver checks this for `parse` and `accept`, each with and without `ignore_comments`. The +shared code is `check_input_adapters()` in `tests/src/fuzzer_common.hpp`. + +The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`. ## Corpus creation diff --git a/tests/src/fuzzer-parse_bjdata.cpp b/tests/src/fuzzer-parse_bjdata.cpp index 92e1bed4c..0387e3241 100644 --- a/tests/src/fuzzer-parse_bjdata.cpp +++ b/tests/src/fuzzer-parse_bjdata.cpp @@ -47,12 +47,30 @@ dumps is stable under exactly the same values that break operator==. The unit tests run the same checks on a fixed corpus (see the "BJData round-trip invariants" test case), so keep both in sync. +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_bjdata for check_input_adapters() +struct read_bjdata +{ + template + json operator()(Args&& ... args) const + { + return json::from_bjdata(std::forward(args)...); + } +}; +} // namespace + // value-stable comparison for the round-trip checks below; see the note // above on why this compares dump()s rather than the json values directly static bool is_value_stable(const json& lhs, const json& rhs) @@ -126,6 +144,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bjdata()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_bon8.cpp b/tests/src/fuzzer-parse_bon8.cpp index 792606d05..bec3e2f93 100644 --- a/tests/src/fuzzer-parse_bon8.cpp +++ b/tests/src/fuzzer-parse_bon8.cpp @@ -17,44 +17,33 @@ array data, it performs the following steps: - j2 = from_bon8(vec) - assert(to_bon8(j2) == vec) -It also checks that reading the data from a stream, which reads strings byte by -byte, gives the same value or error as reading it from contiguous memory, which -copies strings in bulk. +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ -#include #include "fuzzer_common.hpp" namespace { -// the serialization of the value read from @a input, or the error message -template -std::string read_bon8(InputType&& input) +// json::from_bon8 for check_input_adapters() +struct read_bon8 { - try + template + json operator()(Args&& ... args) const { - const auto vec = json::to_bon8(json::from_bon8(std::forward(input))); - return {vec.begin(), vec.end()}; + return json::from_bon8(std::forward(args)...); } - catch (const json::exception& e) - { - return e.what(); - } -} +}; } // namespace // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - // contiguous and stream input must be read alike - { - std::istringstream stream(std::string(reinterpret_cast(data), size)); - assert(read_bon8(std::vector(data, data + size)) == read_bon8(stream)); - } - std::vector const vec1(data, data + size); // step 0: parse input without exceptions @@ -106,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bon8()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_bson.cpp b/tests/src/fuzzer-parse_bson.cpp index 548c30f38..33179de2b 100644 --- a/tests/src/fuzzer-parse_bson.cpp +++ b/tests/src/fuzzer-parse_bson.cpp @@ -17,12 +17,30 @@ array data, it performs the following steps: - j2 = from_bson(vec) - assert(to_bson(j2) == vec) +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_bson for check_input_adapters() +struct read_bson +{ + template + json operator()(Args&& ... args) const + { + return json::from_bson(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_bson()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_cbor.cpp b/tests/src/fuzzer-parse_cbor.cpp index 6d995dc6e..12e7d4104 100644 --- a/tests/src/fuzzer-parse_cbor.cpp +++ b/tests/src/fuzzer-parse_cbor.cpp @@ -17,12 +17,30 @@ array data, it performs the following steps: - j2 = from_cbor(vec) - assert(to_cbor(j2) == vec) +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_cbor for check_input_adapters() +struct read_cbor +{ + template + json operator()(Args&& ... args) const + { + return json::from_cbor(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_cbor()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_json.cpp b/tests/src/fuzzer-parse_json.cpp index 31a811a6c..d656bfb87 100644 --- a/tests/src/fuzzer-parse_json.cpp +++ b/tests/src/fuzzer-parse_json.cpp @@ -17,6 +17,12 @@ array data, it performs the following steps: - j2 = parse(s1) - s2 = serialize(j2) - assert(s1 == s2) +- for ignore_comments = false and true: assert(parse(data) and accept(data) + give the same result for a stream, a non-contiguous iterator range, and a + FILE* as for the pointer range) + +The pointer range takes the bulk fast paths, the other input adapters read byte +by byte; see check_input_adapters(). The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. @@ -24,6 +30,32 @@ drivers. #include "fuzzer_common.hpp" +namespace +{ +// json::parse and json::accept for check_input_adapters() +struct parse_json +{ + bool ignore_comments; + + template + json operator()(Args&& ... args) const + { + return json::parse(std::forward(args)..., nullptr, true, ignore_comments); + } +}; + +struct accept_json +{ + bool ignore_comments; + + template + bool operator()(Args&& ... args) const + { + return json::accept(std::forward(args)..., ignore_comments); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -76,6 +108,16 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + for (const bool ignore_comments : + { + false, true + }) + { + check_input_adapters(data, size, parse_json{ignore_comments}); + check_input_adapters(data, size, accept_json{ignore_comments}); + } + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_msgpack.cpp b/tests/src/fuzzer-parse_msgpack.cpp index a5e2e346f..192cbffd8 100644 --- a/tests/src/fuzzer-parse_msgpack.cpp +++ b/tests/src/fuzzer-parse_msgpack.cpp @@ -17,12 +17,30 @@ array data, it performs the following steps: - j2 = from_msgpack(vec) - assert(to_msgpack(j2) == vec) +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_msgpack for check_input_adapters() +struct read_msgpack +{ + template + json operator()(Args&& ... args) const + { + return json::from_msgpack(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -77,6 +95,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_msgpack()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer-parse_ubjson.cpp b/tests/src/fuzzer-parse_ubjson.cpp index 641526c90..472857a07 100644 --- a/tests/src/fuzzer-parse_ubjson.cpp +++ b/tests/src/fuzzer-parse_ubjson.cpp @@ -26,12 +26,30 @@ array data, it performs the following steps: The unit tests run the same checks on a fixed corpus (see the "UBJSON round-trip invariants" test case), so keep both in sync. +Finally, it reads the data from a stream, a non-contiguous iterator range, and +a FILE*, which read byte by byte, and checks that each gives the same value or +error (id, message, byte position) as the pointer range, which takes the bulk +fast paths (see check_input_adapters()). + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ #include "fuzzer_common.hpp" +namespace +{ +// json::from_ubjson for check_input_adapters() +struct read_ubjson +{ + template + json operator()(Args&& ... args) const + { + return json::from_ubjson(std::forward(args)...); + } +}; +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { @@ -96,6 +114,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) assert(parsed || noexcept_threw || j_noexcept.is_discarded()); } + // step 3: the other input adapters must read the data alike + check_input_adapters(data, size, read_ubjson()); + // return 0 - non-zero return values are reserved for future use return 0; } diff --git a/tests/src/fuzzer_common.hpp b/tests/src/fuzzer_common.hpp index 4b6c9289c..8761b3a35 100644 --- a/tests/src/fuzzer_common.hpp +++ b/tests/src/fuzzer_common.hpp @@ -11,6 +11,14 @@ #pragma once #include // assert +#include // size_t +#include // uint8_t +#include // FILE, fmemopen, fclose +#include // list +#include // istringstream +#include // string +#include // forward +#include // vector #include using nlohmann::json; // NOLINT(google-global-names-in-headers): shared by all fuzzer drivers @@ -49,3 +57,89 @@ json parse_without_exceptions(Parse parse, bool& threw) } return {}; } + +// the outcome of reading an input with exceptions: the serialization of the +// value, or the id, message, and (for parse errors) byte of the exception +struct read_outcome +{ + std::string value{}; + int id = 0; + std::string what{}; + std::size_t byte = 0; + + bool operator==(const read_outcome& rhs) const + { + return value == rhs.value && id == rhs.id && what == rhs.what && byte == rhs.byte; + } +}; + +inline std::string outcome_value(const json& j) +{ + return j.dump(-1, ' ', false, json::error_handler_t::keep); +} + +inline std::string outcome_value(bool b) +{ + return b ? "true" : "false"; +} + +template +read_outcome read_with(const Read& parse, Args&& ... args) +{ + read_outcome result; + try + { + result.value = outcome_value(parse(std::forward(args)...)); + } + catch (const json::parse_error& e) + { + result.id = e.id; + result.what = e.what(); + result.byte = e.byte; + } + catch (const json::exception& e) + { + result.id = e.id; + result.what = e.what(); + } + return result; +} + +// The contiguous input (a pointer range) takes the bulk fast paths; the other +// input adapters read byte by byte. Reads @a data through each of them and +// checks that all give the same value, or the same exception with the same +// id, message, and byte position. +// +// @a parse is called with either an iterator range or a single input (stream +// or FILE*) and returns the parsed value (or the result of accept()). +template +void check_input_adapters(const std::uint8_t* data, std::size_t size, const Read& parse) +{ + const read_outcome expected = read_with(parse, data, data + size); + + // stream adapter + { + std::istringstream stream(std::string(reinterpret_cast(data), size)); + assert(read_with(parse, stream) == expected); + } + + // iterator adapter on a non-contiguous range + { + const std::list list(data, data + size); + assert(read_with(parse, list.begin(), list.end()) == expected); + } + +#if defined(__unix__) || defined(__APPLE__) + // FILE* adapter; fmemopen may refuse an empty buffer + { + std::vector buffer(data, data + size); + std::FILE* file = buffer.empty() ? nullptr : fmemopen(buffer.data(), buffer.size(), "rb"); + if (file != nullptr) + { + const read_outcome actual = read_with(parse, file); + std::fclose(file); + assert(actual == expected); + } + } +#endif +} diff --git a/tests/src/unit-algorithms.cpp b/tests/src/unit-algorithms.cpp index 90d6a08c7..2a9506590 100644 --- a/tests/src/unit-algorithms.cpp +++ b/tests/src/unit-algorithms.cpp @@ -9,6 +9,7 @@ #include "doctest_compatibility.h" #include +#include #include using nlohmann::json; diff --git a/tests/src/unit-comparison.cpp b/tests/src/unit-comparison.cpp index 1e409bfb0..4ac43320b 100644 --- a/tests/src/unit-comparison.cpp +++ b/tests/src/unit-comparison.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -1149,3 +1150,157 @@ TEST_CASE("operator<=> of binary values with a different subtype does not depend } } #endif + +TEST_CASE("comparison of scalars (inline fast path)") +{ + const json::number_integer_t neg = -1; + const json::number_integer_t pos = 5; + const json::number_unsigned_t big = (std::numeric_limits::max)(); + + SECTION("same type") + { + CHECK(json(pos) == json(pos)); + CHECK(json(pos) != json(neg)); + CHECK(json(neg) < json(pos)); + CHECK_FALSE(json(pos) < json(neg)); + CHECK_FALSE(json(pos) < json(pos)); + CHECK(json(pos) <= json(pos)); + + CHECK(json(1u) == json(1u)); + CHECK(json(1u) < json(2u)); + CHECK_FALSE(json(2u) < json(1u)); + + CHECK(json(1.5) == json(1.5)); + CHECK(json(1.5) < json(2.5)); + CHECK_FALSE(json(2.5) < json(1.5)); + CHECK(json(-0.0) == json(0.0)); + } + + SECTION("mixed number types") + { + // negative signed vs unsigned must not wrap around + CHECK(json(neg) < json(1u)); + CHECK(json(1u) > json(neg)); + CHECK(json(neg) != json(big)); + CHECK(json(neg) < json(big)); + CHECK_FALSE(json(big) < json(neg)); + + CHECK(json(pos) == json(5u)); + CHECK(json(5u) == json(pos)); + CHECK(json(pos) < json(6u)); + CHECK(json(pos) == json(5.0)); + CHECK(json(5.0) == json(pos)); + CHECK(json(5u) == json(5.0)); + CHECK(json(neg) < json(0.5)); + CHECK(json(0.5) < json(1u)); + } + + SECTION("boolean and null") + { + CHECK(json(true) == json(true)); + CHECK(json(true) != json(false)); + CHECK(json(false) < json(true)); + CHECK_FALSE(json(true) < json(false)); + CHECK_FALSE(json(true) < json(true)); + + CHECK(json(nullptr) == json(nullptr)); + CHECK_FALSE(json(nullptr) < json(nullptr)); + CHECK(json(nullptr) <= json(nullptr)); + CHECK(json(nullptr) < json(false)); + CHECK(json(nullptr) != json(false)); + CHECK(json(nullptr) != json(0)); + } + + SECTION("string vs number") + { + CHECK(json("5") != json(5)); + CHECK(json(5) != json("5")); + CHECK(json(5) < json("5")); // number < string by type order + CHECK_FALSE(json("5") < json(5)); + CHECK(json(1.5) < json("")); + CHECK(json("a") == json("a")); + CHECK(json("a") < json("b")); + } + + SECTION("NaN") + { + const json nan = std::numeric_limits::quiet_NaN(); + const json one = 1.0; + + CHECK_FALSE(nan == nan); + CHECK(nan != nan); + CHECK_FALSE(nan < nan); + CHECK_FALSE(nan <= nan); + CHECK_FALSE(nan >= nan); + CHECK_FALSE(nan > nan); + + CHECK_FALSE(nan == one); + CHECK_FALSE(one == nan); + CHECK_FALSE(nan < one); + CHECK_FALSE(one < nan); + CHECK_FALSE(nan <= one); + CHECK_FALSE(one <= nan); + CHECK_FALSE(nan >= one); + CHECK_FALSE(one >= nan); + + // NaN against other number types + CHECK_FALSE(nan == json(1)); + CHECK_FALSE(json(1) == nan); + CHECK_FALSE(nan < json(1u)); + CHECK_FALSE(json(1u) < nan); + +#if JSON_HAS_THREE_WAY_COMPARISON + CHECK((nan <=> nan) == std::partial_ordering::unordered); // *NOPAD* + CHECK((nan <=> one) == std::partial_ordering::unordered); // *NOPAD* + CHECK((one <=> nan) == std::partial_ordering::unordered); // *NOPAD* + CHECK((one <=> json(2.0)) == std::partial_ordering::less); // *NOPAD* + CHECK((json(2.0) <=> one) == std::partial_ordering::greater); // *NOPAD* + CHECK((one <=> one) == std::partial_ordering::equivalent); // *NOPAD* + CHECK((json(neg) <=> json(1u)) == std::partial_ordering::less); // *NOPAD* + CHECK((json(nullptr) <=> json(nullptr)) == std::partial_ordering::equivalent); // *NOPAD* + CHECK((json(true) <=> json(false)) == std::partial_ordering::greater); // *NOPAD* +#endif + } + + SECTION("discarded") + { + const json d(json::value_t::discarded); + + CHECK_FALSE(d == d); + CHECK_FALSE(d < d); + CHECK_FALSE(d == json(1)); + CHECK_FALSE(json(1) == d); + CHECK_FALSE(d < json(1)); + CHECK_FALSE(json(1) < d); + CHECK_FALSE(d == json(nullptr)); + CHECK_FALSE(json(nullptr) == d); + +#if JSON_HAS_THREE_WAY_COMPARISON + CHECK((d <=> d) == std::partial_ordering::unordered); // *NOPAD* + CHECK((d <=> json(1)) == std::partial_ordering::unordered); // *NOPAD* + CHECK((json(1) <=> d) == std::partial_ordering::unordered); // *NOPAD* +#endif + } + + SECTION("scalar overloads") + { + CHECK(json(5) == 5); + CHECK(5 == json(5)); + CHECK(json(5) < 6); + CHECK(4 < json(5)); + CHECK(json(1.5) == 1.5); + CHECK(json(true) == true); + CHECK(json(5) != 6); + CHECK(json(1u) == 1.0); + } + + SECTION("structured values still compare") + { + CHECK(json({1, 2}) == json({1, 2})); + CHECK(json({1, 2}) < json({1, 3})); + CHECK(json({{"a", 1}}) == json({{"a", 1}})); + CHECK(json::binary({1, 2}) == json::binary({1, 2})); + CHECK(json(1) < json::array()); + CHECK(json::object() < json::array()); + } +} diff --git a/tests/src/unit-constructor2.cpp b/tests/src/unit-constructor2.cpp index d1ff56452..4896bb1f0 100644 --- a/tests/src/unit-constructor2.cpp +++ b/tests/src/unit-constructor2.cpp @@ -8,6 +8,9 @@ #include "doctest_compatibility.h" +#include +#include + #include using nlohmann::json; @@ -184,3 +187,193 @@ TEST_CASE("other constructors and destructor") } } } + +TEST_CASE("copy constructor and destructor across value types") +{ + SECTION("copy of a scalar is independent of the source") + { + const json src_null = nullptr; + const json src_bool = true; + const json src_int = -7; + const json src_uint = 7u; + const json src_float = 2.5; + + json c_null(src_null); + json c_bool(src_bool); + json c_int(src_int); + json c_uint(src_uint); + json c_float(src_float); + + CHECK(c_null.is_null()); + CHECK(c_bool == true); + CHECK(c_int == -7); + CHECK(c_uint == 7u); + CHECK(c_float == 2.5); + + c_int = 8; + c_float = 3.5; + CHECK(src_int == -7); + CHECK(src_float == 2.5); + } + + SECTION("copy of an owning value, then mutate the copy") + { + const json src_string = "hello"; + const json src_array = {1, 2, 3}; + const json src_object = {{"a", 1}, {"b", {{"c", 2}}}}; + const json src_binary = json::binary({1, 2, 3}); + + json c_string(src_string); + json c_array(src_array); + json c_object(src_object); + json c_binary(src_binary); + + CHECK(c_string == src_string); + CHECK(c_array == src_array); + CHECK(c_object == src_object); + CHECK(c_binary == src_binary); + + c_string.get_ref() += " world"; + c_array.push_back(4); + c_object["b"]["c"] = 3; + c_object["d"] = 4; + c_binary.get_binary().push_back(4); + + CHECK(src_string == "hello"); + CHECK(src_array.size() == 3); + CHECK(src_object["b"]["c"] == 2); + CHECK_FALSE(src_object.contains("d")); + CHECK(src_binary.get_binary().size() == 3); + + CHECK(c_string == "hello world"); + CHECK(c_array.size() == 4); + CHECK(c_object["b"]["c"] == 3); + CHECK(c_binary.get_binary().size() == 4); + } + + SECTION("copy keeps parent pointers valid (JSON_DIAGNOSTICS)") + { + const json src = {{"a", {1, 2}}, {"b", "x"}}; + json c(src); + c["a"].push_back(3); + CHECK(c["a"].size() == 3); + CHECK(src["a"].size() == 2); + } + + SECTION("vector of mixed types: build, copy, and destroy") + { + { + std::vector v; + for (int i = 0; i < 70; ++i) + { + switch (i % 9) + { + case 0: + v.emplace_back(nullptr); + break; + case 1: + v.emplace_back(i % 2 == 0); + break; + case 2: + v.emplace_back(-i); + break; + case 3: + v.emplace_back(static_cast(i)); + break; + case 4: + v.emplace_back(i * 0.5); + break; + case 5: + v.emplace_back(std::string(static_cast(i), 'x')); + break; + case 6: + v.emplace_back(json::array({i, "s", nullptr})); + break; + case 7: + v.emplace_back(json::object({{"k", i}})); + break; + default: + v.emplace_back(json::binary({1, 2, 3})); + break; + } + } + REQUIRE(v.size() == 70); + + std::vector copy(v); + CHECK(copy == v); + + // mutating the copy leaves the original alone + copy[5] = "changed"; + copy[6].push_back(1); + CHECK(copy != v); + CHECK(v[5] == std::string(5, 'x')); + CHECK(v[6].size() == 3); + + v.erase(v.begin(), v.begin() + 10); + CHECK(v.size() == 60); + } + + // destroying the discarded and default-constructed values + std::vector w(4); + w.emplace_back(json::value_t::discarded); + w.emplace_back(json::value_t::object); + w.emplace_back(json::value_t::array); + w.emplace_back(json::value_t::string); + w.emplace_back(json::value_t::binary); + CHECK(w.size() == 9); + } +} + +TEST_CASE_TEMPLATE("destructor of nested containers", Json, nlohmann::json, nlohmann::ordered_json) +{ + SECTION("empty containers") + { + { + Json j = Json::array(); + } + { + Json j = Json::object(); + } + { + Json j = Json::array({Json::array(), Json::object(), Json::array({Json::array()})}); + } + } + + SECTION("leaves mixed with nested containers") + { + Json j = Json::object(); + j["leaf"] = 1; + j["str"] = "text"; + j["empty_a"] = Json::array(); + j["empty_o"] = Json::object(); + j["arr"] = Json::array({1, "a", Json::array({1, 2}), Json::object({{"k", Json::array({Json::object()})}}), nullptr}); + j["obj"] = Json::object({{"x", Json::object({{"y", Json::array({1, Json::object({{"z", 1}})})}})}, {"w", 2}}); + Json copy = j; + CHECK(copy == j); + j = Json::array({std::move(copy), j, 3, Json::array()}); + } + + SECTION("deep nesting") + { + for (int kind = 0; kind < 3; ++kind) + { + Json j = 1; + for (int i = 0; i < 5000; ++i) + { + switch (kind) + { + case 0: + j = Json::array({std::move(j)}); + break; + case 1: + j = Json::object({{"k", std::move(j)}}); + break; + default: + // siblings next to the nested child + j = (i % 2 == 0) ? Json::array({1, Json::array(), std::move(j)}) : Json::object({{"a", 1}, {"b", Json::object()}, {"c", std::move(j)}}); + break; + } + } + } + } +} diff --git a/tests/src/unit-diagnostics.cpp b/tests/src/unit-diagnostics.cpp index 37427587c..f791509b4 100644 --- a/tests/src/unit-diagnostics.cpp +++ b/tests/src/unit-diagnostics.cpp @@ -13,9 +13,11 @@ #endif #define JSON_DIAGNOSTICS 1 +#define JSON_TESTS_PRIVATE #include using nlohmann::json; +using nlohmann::ordered_json; #include #include @@ -631,3 +633,238 @@ TEST_CASE("Better diagnostics past the descent bound of update(), merge_patch() CHECK_THROWS_WITH_AS(p->at("y").at(0), expected_y.c_str(), json::type_error); } } + +namespace +{ +// check that every child's parent pointer points to the containing value +template +bool parents_consistent(const BasicJsonType& j) +{ + bool ok = true; + if (j.is_array() || j.is_object()) + { + for (const auto& child : j) + { + ok = ok && child.m_parent == &j && parents_consistent(child); + } + } + return ok; +} +} // namespace + +TEST_CASE_TEMPLATE("Parent pointers stay consistent", J, json, ordered_json) +{ + SECTION("parse, copy, move, assign, swap") + { + J j = J::parse(R"({"a":[1,{"b":[2,3]},[4]],"c":{"d":{"e":[]}}})"); + CHECK(parents_consistent(j)); + + J copy(j); + CHECK(parents_consistent(copy)); + + J moved(std::move(copy)); + CHECK(parents_consistent(moved)); + + J assigned; + assigned = j; + CHECK(parents_consistent(assigned)); + + J other = {1, 2, {{"k", J::array({1, 2})}}}; + other.swap(assigned); + CHECK(parents_consistent(other)); + CHECK(parents_consistent(assigned)); + } + + SECTION("push_back growth") + { + J arr = J::array(); + for (int i = 0; i < 500; ++i) + { + arr.push_back(J::array({i})); + } + CHECK(parents_consistent(arr)); + + J obj = J::object(); + for (int i = 0; i < 500; ++i) + { + obj.push_back({std::to_string(i), J::array({i})}); + } + CHECK(parents_consistent(obj)); + } + + SECTION("emplace_back and emplace growth") + { + J arr = J::array(); + J obj = J::object(); + for (int i = 0; i < 500; ++i) + { + arr.emplace_back(J::array({i})); + obj.emplace(std::to_string(i), J::array({i})); + } + CHECK(parents_consistent(arr)); + CHECK(parents_consistent(obj)); + } + + SECTION("operator[] growth and at()") + { + J obj = J::object(); + for (int i = 0; i < 500; ++i) + { + obj[std::to_string(i)] = J::array({i}); + CHECK(obj[std::to_string(i / 2)].m_parent == &obj); + } + CHECK(parents_consistent(obj)); + + J arr = J::array(); + for (int i = 0; i < 500; ++i) + { + arr[static_cast(i)] = J::object({{"x", i}}); + } + for (size_t i = 0; i < 500; ++i) + { + CHECK(arr.at(i).m_parent == &arr); + } + CHECK(obj.at("3").m_parent == &obj); + CHECK(parents_consistent(arr)); + CHECK(parents_consistent(obj)); + } +} + +TEST_CASE_TEMPLATE("update() and merge_patch() keep parent pointers and diagnostic paths", J, json, ordered_json) +{ + // enough keys to make the vector behind ordered_json reallocate several times + const auto make_object = [](int first, int last) + { + J o = J::object(); + for (int i = first; i < last; ++i) + { + o[std::to_string(i)] = J::object({{"x", J::array({i})}}); + } + return o; + }; + + // const access, so operator[] cannot repair the parent pointers + const auto check_path = [](const J & j, const std::string & key) + { + const std::string expected = "[json.exception.type_error.304] (/" + key + "/x) cannot use at() with array"; + CHECK_THROWS_WITH_AS(j[key]["x"].at("y"), expected.c_str(), json::type_error); + }; + + SECTION("update(const_reference)") + { + J j = make_object(0, 200); + j.update(make_object(100, 400)); + CHECK(j.size() == 400); + CHECK(parents_consistent(j)); + check_path(j, "0"); + check_path(j, "150"); + check_path(j, "399"); + } + + SECTION("update(const_reference, merge_objects)") + { + J j = make_object(0, 200); + j["50"]["y"] = 1; + J other = make_object(100, 400); + other["50"] = J::object({{"z", J::array({1})}}); + j.update(other, true); + CHECK(j.size() == 400); + CHECK(j["50"].size() == 3); + CHECK(parents_consistent(j)); + check_path(j, "0"); + check_path(j, "399"); + } + + SECTION("update(first, last)") + { + J j = make_object(0, 200); + J other = make_object(100, 400); + j.update(other.begin(), other.end()); + CHECK(j.size() == 400); + CHECK(parents_consistent(j)); + check_path(j, "0"); + check_path(j, "399"); + + J k = make_object(0, 200); + k.update(other.begin(), other.end(), true); + CHECK(k.size() == 400); + CHECK(parents_consistent(k)); + check_path(k, "399"); + } + + SECTION("merge_patch()") + { + J j = make_object(0, 200); + J patch = make_object(100, 400); + patch["0"] = nullptr; + j.merge_patch(patch); + CHECK(j.size() == 399); + CHECK_FALSE(j.contains("0")); + CHECK(parents_consistent(j)); + check_path(j, "1"); + check_path(j, "399"); + } +} + +namespace +{ +// an object type that, like a sorted flat map, moves its members when a key is +// added even if its capacity does not change; it has a capacity() member, but +// is neither node-based nor nlohmann::ordered_map, so the library must not +// trust its capacity to tell whether the members moved +template, + class Allocator = std::allocator>> + struct relocating_map : nlohmann::ordered_map +{ + using base_type = nlohmann::ordered_map; + using typename base_type::iterator; + using typename base_type::Container; + using base_type::base_type; + + template + std::pair emplace(Args&& ... args) + { + auto result = base_type::emplace(std::forward(args)...); + const auto index = result.first - this->begin(); + if (result.second) + { + // move every member to a new buffer of the same capacity + Container relocated; + relocated.reserve(this->capacity()); + for (auto& member : *this) + { + relocated.emplace_back(member.first, std::move(member.second)); + } + this->Container::swap(relocated); + } + return {this->begin() + index, result.second}; + } +}; + +using relocating_json = nlohmann::basic_json; +} // namespace + +TEST_CASE("Parent pointers stay consistent for object types not known to keep their members in place") +{ + relocating_json obj = relocating_json::object(); + for (int i = 0; i < 100; ++i) + { + obj[std::to_string(i)] = relocating_json::array({i}); + obj.emplace(std::to_string(i + 1000), relocating_json::array({i})); + } + CHECK(parents_consistent(obj)); + + obj.erase("0"); + CHECK(parents_consistent(obj)); + + relocating_json other = relocating_json::object(); + for (int i = 0; i < 100; ++i) + { + other[std::to_string(i + 2000)] = relocating_json::array({i}); + } + obj.update(other); + CHECK(parents_consistent(obj)); + + obj["1"].push_back(42); + CHECK_THROWS_WITH_AS(obj["1"].at(5), "[json.exception.out_of_range.401] (/1) array index 5 is out of range", relocating_json::out_of_range&); +} diff --git a/tests/src/unit-iterators1.cpp b/tests/src/unit-iterators1.cpp index b7b0aa9eb..629c6abd7 100644 --- a/tests/src/unit-iterators1.cpp +++ b/tests/src/unit-iterators1.cpp @@ -10,6 +10,8 @@ #define JSON_TESTS_PRIVATE #include +#include +#include using nlohmann::json; TEST_CASE("iterators 1") @@ -1628,3 +1630,108 @@ TEST_CASE("iterators 1") } } } + +TEST_CASE_TEMPLATE("reverse iteration and member access", Json, nlohmann::json, nlohmann::ordered_json) +{ + SECTION("array: -- and rbegin/rend") + { + Json j = {1, 2, 3, 4}; + + auto it = j.end(); + std::vector seen; + while (it != j.begin()) + { + --it; + seen.push_back(it->template get()); + } + CHECK(seen == std::vector {4, 3, 2, 1}); + + auto it2 = j.end(); + auto old = it2--; + CHECK(old == j.end()); + CHECK(*it2 == 4); + + seen.clear(); + for (auto r = j.rbegin(); r != j.rend(); ++r) + { + seen.push_back(r->template get()); + } + CHECK(seen == std::vector {4, 3, 2, 1}); + + seen.clear(); + for (auto r = j.crbegin(); r != j.crend(); ++r) + { + seen.push_back(*r); + } + CHECK(seen == std::vector {4, 3, 2, 1}); + } + + SECTION("object: -- and rbegin/rend") + { + Json j = {{"a", 1}, {"b", 2}, {"c", 3}}; + // order of keys depends on the object type; use the forward order as reference + std::vector fwd; + for (auto it = j.begin(); it != j.end(); ++it) + { + fwd.push_back(it.key()); + } + std::vector rev(fwd.rbegin(), fwd.rend()); + + std::vector seen; + auto it = j.end(); + while (it != j.begin()) + { + --it; + seen.push_back(it.key()); + } + CHECK(seen == rev); + + auto it2 = j.end(); + auto old = it2--; + CHECK(old == j.end()); + CHECK(it2.key() == rev.front()); + + seen.clear(); + for (auto r = j.rbegin(); r != j.rend(); ++r) + { + seen.push_back(r.key()); + } + CHECK(seen == rev); + } + + SECTION("object: operator-> reads and writes the mapped value") + { + Json j = {{"a", 1}, {"s", "str"}}; + auto it = j.find("s"); + REQUIRE(it != j.end()); + CHECK(it->is_string()); + CHECK(it->size() == 1); + *it = Json::array({1, 2}); + CHECK(it->is_array()); + it->push_back(3); + CHECK(j["s"].size() == 3); + + const Json& cj = j; + auto cit = cj.find("a"); + REQUIRE(cit != cj.end()); + CHECK(cit->is_number_integer()); + CHECK(cit->template get() == 1); + } + + SECTION("primitives: -- and ->") + { + Json j = 42; + auto it = j.end(); + --it; + CHECK(it == j.begin()); + CHECK(it->template get() == 42); + } + + SECTION("comparing iterators of different containers throws") + { + Json a = {1, 2}; + Json b = {1, 2}; + CHECK_THROWS_WITH_AS(a.begin() == b.begin(), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", nlohmann::json::invalid_iterator); + CHECK_THROWS_WITH_AS(a.begin() < b.begin(), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers", nlohmann::json::invalid_iterator); + } +} diff --git a/tests/src/unit-json_patch.cpp b/tests/src/unit-json_patch.cpp index 9e83edbcf..67ffee6dd 100644 --- a/tests/src/unit-json_patch.cpp +++ b/tests/src/unit-json_patch.cpp @@ -2066,3 +2066,40 @@ TEST_CASE("JSON patch - every operation on ordered_json") } } } + +TEST_CASE_TEMPLATE("JSON patch - diff round trip on hand-made documents", Json, nlohmann::json, nlohmann::ordered_json) +{ + const auto roundtrip = [](const Json & a, const Json & b) + { + CAPTURE(a.dump()) + CAPTURE(b.dump()) + const Json p = Json::diff(a, b); + CHECK(a.patch(p) == b); + CHECK(Json::diff(b, b).empty()); + }; + + SECTION("reordered keys") + { + const Json a = Json::parse(R"({"x":1,"y":2,"z":{"p":1,"q":2}})"); + const Json b = Json::parse(R"({"z":{"q":2,"p":1},"y":2,"x":1})"); + roundtrip(a, b); + roundtrip(b, a); + } + + SECTION("nested add, remove and replace") + { + const Json a = Json::parse(R"({"a":[1,2,{"k":"v","gone":true}],"b":{"c":{"d":1}},"e":"s"})"); + const Json b = Json::parse(R"({"a":[1,3,{"k":"w","new":[]}],"b":{"c":{"d":[1,2]}},"f":null})"); + roundtrip(a, b); + roundtrip(b, a); + } + + SECTION("arrays of different length and type changes") + { + roundtrip(Json::parse("[1,2,3,4,5]"), Json::parse("[1,3]")); + roundtrip(Json::parse("[1]"), Json::parse("[1,[2],{\"a\":3}]")); + roundtrip(Json::parse("[]"), Json::parse("{}")); + roundtrip(Json::parse("{\"a\":[]}"), Json::parse("{\"a\":{}}")); + roundtrip(Json(1), Json::parse("[1]")); + } +} diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index 658a7fa8d..39c4f790c 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -23,8 +23,14 @@ using nlohmann::json; using namespace nlohmann::literals; // NOLINT(google-build-using-namespace) #endif +#include +#include +#include #include +#include #include +#include +#include TEST_CASE("JSON pointers") { @@ -1071,3 +1077,242 @@ TEST_CASE("flatten of structured values") R"({"/z":"s","/a~1b/~0":1,"/a~1b/k":null,"/a/0":1,"/a/1/c":null,"/a/1/b":null,"/a/2/0/x~0~1/0":true,"/a/2/0/x~0~1/1":null,"/a/2/0/w":2})"); } } + +TEST_CASE("flatten of ordered_json") +{ + using nlohmann::ordered_json; + + SECTION("keeps the document order of the leaves") + { + const ordered_json j = ordered_json::parse(R"({ + "z": 1, "a": {"y": [true, null, {}], "b": "x", "": []}, + "m~n": 2.5, "o/p": {"k": {}}, "0": [[1], [2, 3]] + })"); + + const ordered_json flat = j.flatten(); + std::vector keys; + for (const auto& el : flat.items()) + { + keys.push_back(el.key()); + } + const std::vector expected = + { + "/z", "/a/y/0", "/a/y/1", "/a/y/2", "/a/b", "/a/", "/m~0n", "/o~1p/k", "/0/0/0", "/0/1/0", "/0/1/1" + }; + CHECK(keys == expected); + CHECK(flat.size() == expected.size()); + CHECK(flat["/m~0n"] == 2.5); + CHECK(flat["/a/y/2"].is_null()); + } + + SECTION("round-trips and agrees with json") + { + ordered_json j; + for (int i = 0; i < 200; ++i) + { + j["k" + std::to_string(i)] = {{"v", i}, {"l", {i, i + 1, "s"}}}; + } + const ordered_json flat = j.flatten(); + CHECK(flat.size() == 800); + CHECK(flat.unflatten() == j); + + // same entries as for a sorted object + const nlohmann::json plain = nlohmann::json::parse(j.dump()); + CHECK(nlohmann::json::parse(flat.dump()) == plain.flatten()); + } + + SECTION("primitive and empty root") + { + CHECK(ordered_json(1).flatten() == ordered_json({{"", 1}})); + CHECK(ordered_json::array().flatten() == ordered_json({{"", nullptr}})); + } +} +namespace +{ +// reference implementation of the array index rules (RFC 6901, Sect. 4) based on strtoull +enum class index_class +{ + ok, + leading_zero, + not_a_number, + unresolved, + exceeds_size_type +}; + +index_class classify_index(const std::string& s, std::size_t& idx) +{ + if (s.size() > 1 && s[0] == '0') + { + return index_class::leading_zero; + } + if (!s.empty() && s != "-" && !(s[0] >= '0' && s[0] <= '9')) + { + return index_class::not_a_number; + } + char* end = nullptr; + errno = 0; + const unsigned long long res = std::strtoull(s.c_str(), &end, 10); // NOLINT(runtime/int) + if (end == s.c_str() || errno == ERANGE || static_cast(end - s.c_str()) != s.size()) + { + return index_class::unresolved; + } + if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) + { + return index_class::exceeds_size_type; + } + idx = static_cast(res); + return index_class::ok; +} + +// the observable behavior of json_pointer::at() and contains() for an array index token +void check_index_token(const std::string& token) +{ + const json array = {10, 20, 30}; + const json::json_pointer ptr("/" + token); + + std::size_t idx = 0; + const auto cls = classify_index(token, idx); + const bool exists = cls == index_class::ok && idx < array.size(); + + CAPTURE(token) + CHECK(array.contains(ptr) == (exists && token != "-")); + + if (token == "-") + { + CHECK_THROWS_AS(array.at(ptr), json::out_of_range&); + return; + } + + switch (cls) + { + case index_class::leading_zero: + CHECK_THROWS_AS(array.at(ptr), json::parse_error&); + break; + case index_class::not_a_number: + CHECK_THROWS_AS(array.at(ptr), json::parse_error&); + break; + case index_class::unresolved: + case index_class::exceeds_size_type: + CHECK_THROWS_AS(array.at(ptr), json::out_of_range&); + break; + case index_class::ok: + default: + if (exists) + { + CHECK(array.at(ptr) == array[idx]); + } + else + { + CHECK_THROWS_AS(array.at(ptr), json::out_of_range&); + } + break; + } +} +} // namespace + +TEST_CASE("JSON pointer array index tokens match strtoull semantics") +{ + SECTION("boundary values") + { + const std::string max_digits = std::to_string((std::numeric_limits::max)()); + const std::vector tokens = + { + "", "-", "0", "1", "2", "3", "00", "01", "007", "+1", "-1", " 1", "1 ", "1a", "a1", "0x1", "1e1", "1.0", + "999999999", "1000000000", "99999999999999999", "999999999999999999", "1000000000000000000", + "9999999999999999999", "18446744073709551614", max_digits, "18446744073709551616", + "99999999999999999999999", "000000000000000000000" + }; + for (const auto& token : tokens) + { + check_index_token(token); + } + } + + SECTION("random tokens") + { + std::mt19937 gen(42); + const std::string alphabet = "0123456789012345678901234567890123456789-+ ex"; + for (int i = 0; i < 5000; ++i) + { + std::string token(static_cast(gen() % 23), '0'); + for (auto& c : token) + { + c = alphabet[gen() % alphabet.size()]; + } + check_index_token(token); + } + } +} + +TEST_CASE("JSON pointer operations on pointers with escaped tokens") +{ + const std::vector strings = + { + "", "/", "//", "/a", "/a/b", "/a~1b", "/~0", "/~01", "/~10", "/a~0~1/b/", "/0/1/2", "/ ", "/a/", "/~1/~0" + }; + + for (const auto& str : strings) + { + CAPTURE(str) + const json::json_pointer ptr(str); + CHECK(ptr.to_string() == str); + CHECK(ptr / "x" / "y" / std::size_t{3} == json::json_pointer(str + "/x/y/3")); + CHECK(ptr.parent_pointer() == (str.empty() ? ptr : json::json_pointer(str.substr(0, str.rfind('/'))))); + + // comparison with a string, in place, and with every other pointer +#if !defined(JSON_TEST_DEPRECATED_FUNCTIONS_DELETED) + for (const auto& other : strings) + { + CAPTURE(other) + CHECK((ptr == other) == (str == other)); + CHECK((other == ptr) == (str == other)); + CHECK((ptr != other) == (str != other)); + } + CHECK_THROWS_AS(ptr == "x", json::parse_error&); + CHECK_THROWS_AS(ptr == "/~", json::parse_error&); + CHECK_THROWS_AS(ptr == "/b/~2", json::parse_error&); +#endif + } +} + +TEST_CASE("JSON pointer lookups") +{ + json j = {{"a", {{"b", {1, 2, 3}}}}, {"", 7}, {"~/", {{"x", nullptr}}}}; + const json& cj = j; + + CHECK(cj[json::json_pointer("/a/b/2")] == 3); + CHECK(cj.at(json::json_pointer("/a/b/1")) == 2); + CHECK(cj.at(json::json_pointer("/")) == 7); + CHECK(cj.contains(json::json_pointer("/~0~1/x"))); + CHECK(!cj.contains(json::json_pointer("/a/c"))); + CHECK(cj.value(json::json_pointer("/a/b/0"), 5) == 1); + CHECK(cj.value(json::json_pointer("/a/b/3"), 5) == 5); + CHECK(cj.value(json::json_pointer("/a/zz"), 5) == 5); + + CHECK_THROWS_AS(cj.at(json::json_pointer("/a/c")), json::out_of_range&); + CHECK_THROWS_AS(cj.at(json::json_pointer("/a/b/3")), json::out_of_range&); + + // non-const access reuses existing entries and creates missing ones + j[json::json_pointer("/a/b/1")] = 20; + j[json::json_pointer("/n/m")] = 1; + CHECK(j["a"]["b"] == json({1, 20, 3})); + CHECK(j["n"]["m"] == 1); +} + +TEST_CASE("JSON patch with moves and copies") +{ + json doc = {{"a", {{"b", {1, 2, 3}}}}, {"c", {{"d", "x"}}}}; + + CHECK(doc.patch(json::parse(R"([{"op":"move","from":"/a/b","path":"/c/e"}])")) == + json::parse(R"({"a":{},"c":{"d":"x","e":[1,2,3]}})")); + CHECK(doc.patch(json::parse(R"([{"op":"copy","from":"/a/b","path":"/c/e"}])")) == + json::parse(R"({"a":{"b":[1,2,3]},"c":{"d":"x","e":[1,2,3]}})")); + CHECK(doc.patch(json::parse(R"([{"op":"move","from":"/a/b/0","path":"/a/b/2"}])")) == + json::parse(R"({"a":{"b":[2,3,1]},"c":{"d":"x"}})")); + + // a failing move leaves a copy-patched document untouched + CHECK_THROWS_AS(doc.patch(json::parse(R"([{"op":"move","from":"","path":""}])")), json::out_of_range&); + json inplace = doc; + CHECK_THROWS_AS(inplace.patch_inplace(json::parse(R"([{"op":"move","from":"","path":""}])")), json::out_of_range&); + CHECK(inplace == doc); +} diff --git a/tests/src/unit-same-number-types.cpp b/tests/src/unit-same-number-types.cpp new file mode 100644 index 000000000..ba42710c1 --- /dev/null +++ b/tests/src/unit-same-number-types.cpp @@ -0,0 +1,201 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include + +#include +#include +#include +#include +#include + +// basic_json with the same type for number_integer_t and number_unsigned_t +// (#2573): every integer is stored as number_integer + +namespace +{ + +template +using same_types_json = nlohmann::json::with_integers_t; + +using json64 = same_types_json; +using json32 = same_types_json; + +} // namespace + +TEST_CASE_TEMPLATE("same type for number_integer_t and number_unsigned_t", Json, json64, json32) +{ + using integer_t = typename Json::number_integer_t; + using limits = std::numeric_limits; + + SECTION("parsing stores integers as number_integer") + { + const Json j = Json::parse("[0, 1, -1, 2147483647, -2147483648]"); + for (const auto& element : j) + { + CHECK(element.is_number_integer()); + CHECK(!element.is_number_unsigned()); + CHECK(element.type() == nlohmann::detail::value_t::number_integer); + } + CHECK(j[1].template get() == 1); + CHECK(j[2].template get() == -1); + CHECK(j[3].template get() == 2147483647); + CHECK(j[4].template get() == -2147483648LL); + } + + SECTION("integers above the maximum are parsed as floats") + { + // a positive literal is parsed as unsigned first; it must not wrap + // around to a negative number_integer_t + const auto above_max = std::to_string(static_cast(limits::max()) + 1); + const Json j1 = Json::parse(above_max); + CHECK(j1.is_number_float()); + CHECK(j1.template get() == static_cast(static_cast(limits::max()) + 1)); + + const Json j2 = Json::parse("18446744073709551615"); + CHECK(j2.is_number_float()); + CHECK(j2.template get() == 18446744073709551615.0); + + const Json j3 = Json::parse(std::to_string(limits::max())); + CHECK(j3.is_number_integer()); + CHECK(j3.template get() == limits::max()); + } + + SECTION("serialization of the extremes") + { + CHECK(Json(limits::max()).dump() == std::to_string(limits::max())); + // the absolute value of the minimum does not fit into number_unsigned_t + CHECK(Json(limits::min()).dump() == std::to_string(limits::min())); + CHECK(Json::parse(std::to_string(limits::min())) == Json(limits::min())); + CHECK(Json(0).dump() == "0"); + } + + SECTION("construction from unsigned types") + { + const Json j1 = 5u; + CHECK(j1.is_number_integer()); + CHECK(!j1.is_number_unsigned()); + CHECK(j1 == 5); + + const Json j2 = static_cast(200); + CHECK(j2.is_number_integer()); + CHECK(j2.template get() == 200); + + const Json j3 = std::vector {1, 2, 3}; + CHECK(j3.dump() == "[1,2,3]"); + CHECK(j3[0].is_number_integer()); + + const Json j4 = true; + CHECK(j4.is_boolean()); + } + + SECTION("value_t::number_unsigned creates a number_integer") + { + const Json j(nlohmann::detail::value_t::number_unsigned); + CHECK(j.is_number_integer()); + CHECK(!j.is_number_unsigned()); + CHECK(j == 0); + CHECK(j.dump() == "0"); + } + + SECTION("conversions") + { + Json j = 42; + CHECK(j.template get() == 42u); + CHECK(j.template get() == 42u); + CHECK(j.template get() == 42); + CHECK(j.template get() == 42.0); + CHECK(j.template get_ptr() != nullptr); + CHECK(*j.template get_ptr() == 42); + CHECK(j.template get_ref() == 42); + j.template get_ref() = 43; + CHECK(j == 43); + + integer_t i = 0; + j.get_to(i); + CHECK(i == 43); + + const Json object = {{"a", 1}}; + CHECK(object.value("a", 0u) == 1u); + CHECK(object.value("b", 7u) == 7u); + } + + SECTION("conversion from a basic_json with distinct types") + { + const nlohmann::json j = {1, 2u, -3, 4.5}; + CHECK(j[1].is_number_unsigned()); + const Json converted = j; + CHECK(converted.dump() == "[1,2,-3,4.5]"); + CHECK(converted[1].is_number_integer()); + CHECK(!converted[1].is_number_unsigned()); + + const nlohmann::json back = converted; + CHECK(back == j); + } + + SECTION("comparison and hashing") + { + const Json j1 = 1; + const Json j2 = 2u; + CHECK(j1 < j2); + CHECK(j1 != j2); + CHECK(j2 == 2); + CHECK(j2 == 2u); + CHECK(j1 < 1.5); + CHECK(Json(-1) < Json(0u)); + CHECK(std::hash {}(Json(5u)) == std::hash {}(Json(5))); + } + + SECTION("JSON Patch, diff, and flatten") + { + const Json source = Json::parse(R"({"a": [1, 2, 3], "b": 4})"); + const Json target = Json::parse(R"({"a": [1, 5], "c": 6})"); + CHECK(source.patch(Json::diff(source, target)) == target); + CHECK(source.flatten().unflatten() == source); + } + + SECTION("binary formats") + { + const Json j = {{"min", limits::min()}, {"max", limits::max()}, {"zero", 0}, {"small", 100}, {"negative", -100}, {"float", 1.5}}; + + CHECK(Json::from_cbor(Json::to_cbor(j)) == j); + CHECK(Json::from_msgpack(Json::to_msgpack(j)) == j); + CHECK(Json::from_ubjson(Json::to_ubjson(j)) == j); + CHECK(Json::from_bjdata(Json::to_bjdata(j)) == j); + CHECK(Json::from_bson(Json::to_bson(j)) == j); + CHECK(Json::from_bon8(Json::to_bon8(j)) == j); + + for (const auto& element : Json::from_cbor(Json::to_cbor(j))) + { + CHECK(!element.is_number_unsigned()); + } + + // CBOR unsigned integer 2^64-1 does not fit and is read as float + const std::vector cbor_max = {0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + const Json from_cbor = Json::from_cbor(cbor_max); + CHECK(from_cbor.is_number_float()); + CHECK(from_cbor.template get() == 18446744073709551615.0); + + // MessagePack uint 8 is read as number_integer + const std::vector msgpack_uint8 = {0xCC, 0xC8}; + const Json from_msgpack = Json::from_msgpack(msgpack_uint8); + CHECK(from_msgpack.is_number_integer()); + CHECK(from_msgpack == 200); + } +} + +TEST_CASE("same type for number_integer_t and number_unsigned_t: type traits") +{ + CHECK(std::is_same, nlohmann::detail::same_as_number_integer_type>::value); + CHECK(std::is_same, std::uint64_t>::value); + CHECK(nlohmann::detail::is_compatible_number_integer_type::value); + CHECK(!nlohmann::detail::is_compatible_number_integer_type::value); + CHECK(!nlohmann::detail::is_compatible_number_integer_type::value); +} diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index c56c4f5c1..4913c30a6 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -20,8 +20,11 @@ using nlohmann::json; #include +#include #include #include +#include +#include #include "test_utils.hpp" @@ -923,66 +926,68 @@ TEST_CASE("serialization boundary values for the string buffer") // mid-string once fewer than 13 bytes remain (`string_buffer.size() - bytes // < 13`), 13 being one more than the most a single code point can ever // write at once (a surrogate pair: two back-to-back "\uXXXX" escapes, 12 - // bytes). Every write into string_buffer that this check protects happens - // in steps of 2 (a simple "\\x" escape) or 6 (one "\uXXXX" unit), so - // `bytes` only ever takes even values at the point the check runs - the - // tightest values actually reachable are therefore 498 (512 - 498 == 14, - // one simple escape away from the threshold) and 500 (512 - 500 == 12, - // where the flush fires immediately and resets bytes to 0). + // bytes). Characters below 0x60 (control characters, '"' and '\\') are + // escaped straight into the write buffer and never reach string_buffer, so + // under ensure_ascii it fills in steps of 6 (one "\uXXXX" unit for a + // character below U+10000) or 12 (a surrogate pair). With 2-byte characters + // such as U+00E4, `bytes` takes multiples of 6 at the point the check runs; + // the tightest values reachable are therefore 498 (512 - 498 == 14, after + // 83 characters) and 504 (512 - 504 == 8, after 84 characters, where the + // flush fires immediately and resets bytes to 0). - SECTION("a run of 2-byte escapes lands bytes on, and one step past, the flush threshold") + SECTION("a run of 6-byte escapes lands bytes on, and one step past, the flush threshold") { for (const int count : { - 249, 250, 251 + 83, 84, 85 }) { CAPTURE(count) - const json j = std::string(static_cast(count), '\n'); + std::string umlauts; std::string expected = "\""; for (int i = 0; i < count; ++i) { - expected += "\\n"; + umlauts += "\xC3\xA4"; + expected += "\\u00e4"; } expected += '"'; - CHECK(j.dump() == expected); + CHECK(json(umlauts).dump(-1, ' ', true) == expected); } } - SECTION("an ASCII prefix leaves the tightest reachable margin before a 12-byte surrogate pair") + SECTION("a prefix leaves the tightest reachable margin before a 12-byte surrogate pair") { // U+1F600 (the "\xF0\x9F\x98\x80" UTF-8 bytes) is dumped under // ensure_ascii as the 12-byte surrogate pair "\ud83d\ude00"; that // write happens in a single step with no intermediate flush check, so // it is the write most exposed by an off-by-one in the "< 13" guard. - // A prefix of 249 newlines leaves exactly 14 bytes of headroom + // A prefix of 83 U+00E4 characters leaves exactly 14 bytes of headroom // (512 - 498), the smallest margin the guard ever actually allows - // into a new code point; 250 newlines instead trigger the guard's own + // into a new code point; 84 characters instead trigger the guard's own // flush first, so the emoji starts from a freshly emptied (512-byte) - // buffer, and 251 repeats that with one more escape already past the + // buffer, and 85 repeats that with one more escape already past the // reset. Together they cover the margin the guard allows landing on, // one step before, and one step after - all must still produce the // identical, correct escapes. for (const int prefix_count : { - 249, 250, 251 + 83, 84, 85 }) { CAPTURE(prefix_count) - const std::string prefix(static_cast(prefix_count), '\n'); - const std::string emoji = "\xF0\x9F\x98\x80"; - const json j = prefix + emoji; - + std::string prefix; std::string expected_prefix; for (int i = 0; i < prefix_count; ++i) { - expected_prefix += "\\n"; + prefix += "\xC3\xA4"; + expected_prefix += "\\u00e4"; } + const std::string emoji = "\xF0\x9F\x98\x80"; + const json j = prefix + emoji; - // newline escaping does not depend on ensure_ascii: only the - // emoji differs (raw UTF-8 bytes vs. a \u-escaped surrogate pair) + // without ensure_ascii, the whole string is copied unchanged std::string expected_raw = "\""; - expected_raw += expected_prefix; + expected_raw += prefix; expected_raw += emoji; expected_raw += '"'; std::string expected_ascii = "\""; @@ -1027,3 +1032,524 @@ TEST_CASE("serialization boundary values for the string buffer") } } } + +namespace +{ +// appends the escaped form of the ASCII character c to out; written +// independently of the table in serializer::ascii_escapes() +void append_escaped_ascii(std::string& out, const char c, const bool ensure_ascii) +{ + const auto byte = static_cast(c); + switch (byte) + { + case '\b': + out += "\\b"; + break; + case '\t': + out += "\\t"; + break; + case '\n': + out += "\\n"; + break; + case '\f': + out += "\\f"; + break; + case '\r': + out += "\\r"; + break; + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + default: + if (byte < 0x20 || (ensure_ascii && byte == 0x7F)) + { + // byte < 0x80, so the first two hex digits are 0 + const char* const hex = "0123456789abcdef"; + out += "\\u00"; + out += hex[byte >> 4U]; + out += hex[byte & 0x0FU]; + } + else + { + out += c; + } + break; + } +} + +// reference escaper for ASCII strings +std::string reference_escape_ascii(const std::string& s, const bool ensure_ascii) +{ + std::string result = "\""; + for (const char c : s) + { + append_escaped_ascii(result, c, ensure_ascii); + } + result += '"'; + return result; +} +} // namespace + +TEST_CASE("characters below 0x60 are escaped from a table straight into the write buffer") +{ + SECTION("every ASCII character at every position") + { + // lengths up to 17 cover the 8-byte strides of the bulk scanner on + // both sides of the character, so the scanner stops at it from every + // offset + for (std::size_t length = 1; length <= 17; ++length) + { + for (std::size_t pos = 0; pos < length; ++pos) + { + for (unsigned int c = 0; c < 0x80; ++c) + { + std::string s(length, 'a'); + s[pos] = static_cast(c); + CAPTURE(length) + CAPTURE(pos) + CAPTURE(c) + const json j = s; + CHECK(j.dump(-1, ' ', false) == reference_escape_ascii(s, false)); + CHECK(j.dump(-1, ' ', true) == reference_escape_ascii(s, true)); + } + } + } + } + + SECTION("runs of escaped characters") + { + std::string s; + for (unsigned int c = 0; c < 0x20; ++c) + { + s += static_cast(c); + } + s += R"("\"\)"; + CHECK(json(s).dump() == reference_escape_ascii(s, false)); + CHECK(json(s).dump(-1, ' ', true) == reference_escape_ascii(s, true)); + } + + SECTION("escapes straddle the end of the write buffer") + { + // the 8-byte store of a table entry needs room for all 8 bytes, even + // when the escape itself is shorter; a prefix of k plain characters + // shifts the 6-byte escapes across every offset of the 1024-byte + // write buffer + for (std::size_t k = 0; k < 8; ++k) + { + CAPTURE(k) + const std::string s = std::string(k, 'a') + std::string(400, '\x01'); + CHECK(json(s).dump() == reference_escape_ascii(s, false)); + CHECK(json::array({s, s}).dump() == "[" + reference_escape_ascii(s, false) + "," + reference_escape_ascii(s, false) + "]"); + } + } + + SECTION("pending \\u escapes of non-ASCII characters are written first") + { + // under ensure_ascii, U+00E4 is \u-escaped into the string buffer; + // a following escape from the table must not overtake it + const json j = "\xC3\xA4\n\xC3\xA4\"\xF0\x9F\x98\x80\\"; + CHECK(j.dump(-1, ' ', true) == "\"\\u00e4\\n\\u00e4\\\"\\ud83d\\ude00\\\\\""); + CHECK(j.dump(-1, ' ', false) == "\"\xC3\xA4\\n\xC3\xA4\\\"\xF0\x9F\x98\x80\\\\\""); + } + + SECTION("escapes after invalid UTF-8 for every error handler") + { + const json j = "a\xFF\n\xC3\""; + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF", json::type_error&); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"a\\n\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::ignore) == "\"a\\n\\\"\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"a\\ufffd\\n\\ufffd\\\"\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\""); + } +} + +namespace +{ +// expected result of dumping a string: the output, or (with +// error_handler_t::strict) the message of the type_error.316 exception +struct expected_dump +{ + bool throws = false; + std::string text{}; // NOLINT(readability-redundant-member-init) +}; + +std::string upper_hex_byte(const unsigned char byte) +{ + const char* const hex = "0123456789ABCDEF"; + return std::string(1, hex[byte >> 4U]) + hex[byte & 0x0FU]; +} + +void append_u_escape(std::string& out, const std::uint32_t code_unit) +{ + const char* const hex = "0123456789abcdef"; + out += "\\u"; + for (int shift = 12; shift >= 0; shift -= 4) + { + out += hex[(code_unit >> static_cast(shift)) & 0x0FU]; + } +} + +void append_replacement_character(std::string& out, const bool ensure_ascii) +{ + out += ensure_ascii ? "\\ufffd" : "\xEF\xBF\xBD"; +} + +// reference serializer for arbitrary byte strings, written from the documented +// behavior of dump() and independently of serializer::dump_escaped(): UTF-8 is +// decoded strictly (well-formed sequences as in Table 3-7 of the Unicode +// Standard), and each ill-formed sequence is handled as the error handler says +expected_dump reference_dump_string(const std::string& s, const bool ensure_ascii, const json::error_handler_t error_handler) +{ + expected_dump result; + std::string& out = result.text; + out = "\""; + + // handle the ill-formed bytes s[begin, end), of which s[index] is the byte + // that could not be decoded; returns false if an exception is expected + const auto ill_formed = [&](const std::size_t begin, const std::size_t end, const std::size_t index) + { + switch (error_handler) + { + case json::error_handler_t::strict: + result.throws = true; + out = "[json.exception.type_error.316] invalid UTF-8 byte at index " + std::to_string(index) + ": 0x" + upper_hex_byte(static_cast(s[index])); + return false; + case json::error_handler_t::ignore: + break; + case json::error_handler_t::replace: + append_replacement_character(out, ensure_ascii); + break; + case json::error_handler_t::keep: + default: + // kept as is, even with ensure_ascii + out.append(s, begin, end - begin); + break; + } + return true; + }; + + std::size_t i = 0; + while (i < s.size()) + { + const auto lead = static_cast(s[i]); + if (lead < 0x80) + { + append_escaped_ascii(out, s[i], ensure_ascii); + ++i; + continue; + } + + // number of bytes of the sequence and the range of its second byte + std::size_t length = 0; + std::uint32_t code_point = 0; + unsigned char lower = 0x80; + unsigned char upper = 0xBF; + if (lead >= 0xC2 && lead <= 0xDF) + { + length = 2; + code_point = lead & 0x1FU; + } + else if (lead >= 0xE0 && lead <= 0xEF) + { + length = 3; + code_point = lead & 0x0FU; + lower = (lead == 0xE0) ? 0xA0 : 0x80; // no overlong encodings + upper = (lead == 0xED) ? 0x9F : 0xBF; // no surrogates + } + else if (lead >= 0xF0 && lead <= 0xF4) + { + length = 4; + code_point = lead & 0x07U; + lower = (lead == 0xF0) ? 0x90 : 0x80; // no overlong encodings + upper = (lead == 0xF4) ? 0x8F : 0xBF; // nothing above U+10FFFF + } + else + { + // a byte that cannot start a sequence is consumed + if (!ill_formed(i, i + 1, i)) + { + return result; + } + ++i; + continue; + } + + std::size_t j = i + 1; + for (; j < i + length; ++j) + { + if (j == s.size()) + { + // the string ends in the middle of the sequence + switch (error_handler) + { + case json::error_handler_t::strict: + result.throws = true; + out = "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x" + upper_hex_byte(static_cast(s.back())); + return result; + case json::error_handler_t::ignore: + break; + case json::error_handler_t::replace: + append_replacement_character(out, ensure_ascii); + break; + case json::error_handler_t::keep: + default: + out.append(s, i, s.size() - i); + break; + } + out += '"'; + return result; + } + + const auto byte = static_cast(s[j]); + if (byte < lower || byte > upper) + { + break; + } + code_point = (code_point << 6U) | (byte & 0x3FU); + lower = 0x80; + upper = 0xBF; + } + + if (j < i + length) + { + // the sequence was abandoned at s[j]; that byte is read again as + // the start of a new character + if (!ill_formed(i, j, j)) + { + return result; + } + i = j; + continue; + } + + if (!ensure_ascii) + { + out.append(s, i, length); + } + else if (code_point <= 0xFFFF) + { + append_u_escape(out, code_point); + } + else + { + append_u_escape(out, 0xD800U + ((code_point - 0x10000U) >> 10U)); + append_u_escape(out, 0xDC00U + ((code_point - 0x10000U) & 0x3FFU)); + } + i = j; + } + + out += '"'; + return result; +} + +std::string hex_dump(const std::string& s) +{ + std::string result; + for (const char c : s) + { + result += upper_hex_byte(static_cast(c)); + result += ' '; + } + return result; +} + +// compares dump() of s with the reference for every error handler, with +// ensure_ascii on and off; CHECKs only on a mismatch to keep the sweep fast; +// error_handler_t::strict can be skipped where the position of the string +// buffer does not matter, as throwing is slow on some platforms +void check_dump_against_reference(const std::string& s, const bool with_strict = true) +{ + const json j = s; + const std::array error_handlers = + { + { + json::error_handler_t::strict, json::error_handler_t::ignore, + json::error_handler_t::replace, json::error_handler_t::keep + } + }; + for (const auto error_handler : error_handlers) + { + if (!with_strict && error_handler == json::error_handler_t::strict) + { + continue; + } + for (const bool ensure_ascii : + { + false, true + }) + { + const expected_dump expected = reference_dump_string(s, ensure_ascii, error_handler); + expected_dump actual; + int id = 0; + try + { + actual.text = j.dump(-1, ' ', ensure_ascii, error_handler); + } + catch (const json::exception& e) + { + actual.throws = true; + actual.text = e.what(); + id = e.id; + } + if (actual.throws != expected.throws || actual.text != expected.text || (actual.throws && id != 316)) + { + const std::string input = hex_dump(s); + CAPTURE(input) + CAPTURE(ensure_ascii) + CAPTURE(static_cast(error_handler)) + CHECK(actual.throws == expected.throws); + CHECK(actual.text == expected.text); + CHECK((!actual.throws || id == 316)); + } + } + } +} +} // namespace + +TEST_CASE("invalid UTF-8 at every position is handled like the reference") +{ + // ill-formed byte sequences; those that end in a truncated sequence are + // also cut short by the filler that follows, or by the end of the string + const std::vector patterns = + { + // stray continuation bytes + "\x80", "\xBF", + // bytes that cannot start a sequence + "\xC0", "\xC1", "\xF5", "\xF6", "\xF7", "\xF8", "\xF9", "\xFA", "\xFB", "\xFC", "\xFD", "\xFE", "\xFF", + // overlong encodings + "\xC0\xAF", "\xE0\x80\x80", "\xF0\x80\x80\x80", + // UTF-16 surrogates + "\xED\xA0\x80", "\xED\xBF\xBF", + // a code point above U+10FFFF + "\xF4\x90\x80\x80", + // sequences cut short by ASCII, a control character, a quote, a + // backslash, DEL, or the lead byte of a well-formed character + "\xC3" "a", "\xE2\x82" "\n", "\xE2\x82\"", "\xF0\x9F\x98\\", "\xC3\x7F", "\xC3\xC3\xA4", "\xE2\xF0\x9F\x98\x80", + // truncated sequences + "\xC3", "\xE2\x82", "\xF0\x9F\x98" + }; + + // well-formed characters of 1, 2, 3, and 4 bytes + const std::vector fillers = {"a", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x98\x80"}; + + const auto make_string = [](const std::string & filler, const std::size_t before, const std::string & pattern, const std::size_t after) + { + std::string s; + for (std::size_t k = 0; k < before; ++k) + { + s += filler; + } + s += pattern; + for (std::size_t k = 0; k < after; ++k) + { + s += filler; + } + return s; + }; + + SECTION("the reference") + { + // the reference itself, against hand-written results + const std::string s = "a\xFF\n\xC3\"\xE0\x80\xF0\x9F\x98\x80\xE2\x82"; + CHECK(reference_dump_string(s, false, json::error_handler_t::strict).text == "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF"); + CHECK(reference_dump_string(s.substr(2), false, json::error_handler_t::strict).text == "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0x22"); + CHECK(reference_dump_string("\xF0\x9F\x98\x80\xE2\x82", true, json::error_handler_t::strict).text == "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x82"); + CHECK(reference_dump_string(s, false, json::error_handler_t::ignore).text == "\"a\\n\\\"\xF0\x9F\x98\x80\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::ignore).text == "\"a\\n\\\"\\ud83d\\ude00\""); + CHECK(reference_dump_string(s, false, json::error_handler_t::replace).text == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\xEF\xBF\xBD\xEF\xBF\xBD\xF0\x9F\x98\x80\xEF\xBF\xBD\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::replace).text == "\"a\\ufffd\\n\\ufffd\\\"\\ufffd\\ufffd\\ud83d\\ude00\\ufffd\""); + CHECK(reference_dump_string(s, false, json::error_handler_t::keep).text == "\"" + std::string("a\xFF\\n\xC3\\\"\xE0\x80\xF0\x9F\x98\x80\xE2\x82") + "\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::keep).text == "\"" + std::string("a\xFF\\n\xC3\\\"\xE0\x80\\ud83d\\ude00\xE2\x82") + "\""); + check_dump_against_reference(s); + } + + SECTION("short strings") + { + // strings of up to 17 bytes put the pattern at every offset of an + // 8-byte stride of the bulk scanners, with whole strides on both + // sides; lengths around 24, 32, and 40 bytes cross the 16- and 32-byte + // strides; error_handler_t::strict is checked for up to 17 bytes of + // ASCII and 8 bytes of other characters only, as throwing is slow on + // some platforms + for (const auto& filler : fillers) + { + for (std::size_t length = 0; length * filler.size() <= 40; ++length) + { + const std::size_t bytes = length * filler.size(); + if (filler.size() == 1 && bytes > 17 && (bytes + 1) % 8 > 2) + { + continue; + } + for (std::size_t pos = 0; pos <= length; ++pos) + { + for (const auto& pattern : patterns) + { + check_dump_against_reference(make_string(filler, pos, pattern, length - pos), bytes <= (filler.size() == 1 ? 17 : 8)); + } + } + } + } + } + + SECTION("strings crossing the end of the string buffer") + { + // characters that are not copied in bulk go through a 512-byte string + // buffer that is written out once fewer than 13 bytes are left; with + // ensure_ascii, each 2-byte character takes 6 bytes of it and each + // 4-byte character 12, so the pattern meets the end of the buffer after + // about 84 or 42 characters (3-byte characters behave like 2-byte + // ones); plain ASCII, copied in bulk, is checked around 512 bytes; + // positions further away are sampled; error_handler_t::strict throws + // at the first ill-formed byte, regardless of the string buffer, and is + // covered above + for (const auto& filler : fillers) + { + if (filler.size() == 3) + { + continue; + } + const std::size_t escaped_size = filler.size() == 1 ? 1 : filler.size() * 3; + const std::size_t middle = 506 / escaped_size; + const std::size_t window = filler.size() == 1 ? 8 : 16; + for (std::size_t pos = 0; pos <= middle + window; pos += (pos + window >= middle) ? 1 : 29) + { + for (const std::size_t after : + { + std::size_t{0}, std::size_t{12} + }) + { + for (const auto& pattern : patterns) + { + check_dump_against_reference(make_string(filler, pos, pattern, after), false); + } + } + } + } + } + + SECTION("repeated invalid sequences at every offset of the string buffer") + { + // ill-formed sequences that are kept or replaced fill the string + // buffer without a bulk copy in between; a prefix of 0 to 12 bytes + // 0xFF (kept as 1 byte, replaced by 3 or 6) shifts them across every + // offset of the end of the buffer + for (const auto& pattern : patterns) + { + for (std::size_t shift = 0; shift <= 12; ++shift) + { + std::string s(shift, '\xFF'); + while (s.size() < 600) + { + s += pattern; + } + check_dump_against_reference(s, shift == 0); + check_dump_against_reference(s + "\xF0\x9F\x98\x80", false); + } + } + } +} diff --git a/tests/src/unit-throw_messages.cpp b/tests/src/unit-throw_messages.cpp new file mode 100644 index 000000000..1c2289626 --- /dev/null +++ b/tests/src/unit-throw_messages.cpp @@ -0,0 +1,135 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include + +// The exceptions in hot core functions are constructed out of line. These tests pin the +// exact what() texts of the sites that moved, so the refactoring cannot change them. + +#define CHECK_MSG(expr, msg) CHECK_THROWS_WITH(expr, msg) + +TEST_CASE("exception messages of out-of-line throw sites") +{ + json arr = {1, 2, 3}; + json arr2 = {4, 5}; + json obj = {{"a", 1}, {"b", 2}}; + json obj2 = {{"c", 3}}; + json num = 1; + + SECTION("iterators") + { + CHECK_MSG((void)(arr.begin() == arr2.begin()), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers"); + CHECK_MSG((void)(arr.begin() < arr2.begin()), "[json.exception.invalid_iterator.212] cannot compare iterators of different containers"); + CHECK_MSG((void)(obj.begin() < obj.end()), "[json.exception.invalid_iterator.213] cannot compare order of object iterators"); + CHECK_MSG(obj.begin() += 1, "[json.exception.invalid_iterator.209] cannot use offsets with object iterators"); + CHECK_MSG((void)(obj.begin() - 1), "[json.exception.invalid_iterator.209] cannot use offsets with object iterators"); + CHECK_MSG((void)obj.begin()[0], "[json.exception.invalid_iterator.208] cannot use operator[] for object iterators"); + CHECK_MSG((void)arr.begin().key(), "[json.exception.invalid_iterator.207] cannot use key() for non-object iterators"); + json null; + CHECK_MSG((void)*null.end(), "[json.exception.invalid_iterator.214] cannot get value"); + } + + SECTION("constructors") + { + CHECK_MSG(json(arr.begin(), arr2.end()), "[json.exception.invalid_iterator.201] iterators are not compatible"); + CHECK_MSG(json(num.begin() + 1, num.end() + 1), "[json.exception.invalid_iterator.204] iterators out of range"); + CHECK_MSG(json(json::initializer_list_t{1, 2}, false, json::value_t::object), "[json.exception.type_error.301] cannot create object from initializer list"); + } + + SECTION("get_ref and operator[]") + { + CHECK_MSG((void)num.get_ref(), "[json.exception.type_error.303] incompatible ReferenceType for get_ref, actual type is number"); + CHECK_MSG((void)num[0], "[json.exception.type_error.305] cannot use operator[] with a numeric argument with number"); + } + + SECTION("at") + { + const json carr = arr; + const json cobj = obj; + CHECK_MSG((void)arr.at(static_cast(10)), "[json.exception.out_of_range.401] array index 10 is out of range"); + CHECK_MSG((void)carr.at(static_cast(3)), "[json.exception.out_of_range.401] array index 3 is out of range"); + CHECK_MSG((void)num.at(static_cast(0)), "[json.exception.type_error.304] cannot use at() with number"); + CHECK_MSG((void)obj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); + CHECK_MSG((void)cobj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); + CHECK_MSG((void)obj.at(std::string("zz")), "[json.exception.out_of_range.403] key 'zz' not found"); + CHECK_MSG((void)num.at("a"), "[json.exception.type_error.304] cannot use at() with number"); + + nlohmann::ordered_json oobj = {{"a", 1}}; + const nlohmann::ordered_json coobj = oobj; + CHECK_MSG((void)oobj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); + CHECK_MSG((void)coobj.at("z"), "[json.exception.out_of_range.403] key 'z' not found"); + CHECK_MSG((void)coobj.at(std::string("zz")), "[json.exception.out_of_range.403] key 'zz' not found"); + } + + SECTION("erase") + { + CHECK_MSG(arr.erase(arr2.begin()), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(num.erase(num.end()), "[json.exception.invalid_iterator.205] iterator out of range"); + CHECK_MSG(arr.erase(arr.begin(), arr2.end()), "[json.exception.invalid_iterator.203] iterators do not fit current value"); + CHECK_MSG(num.erase(num.begin() + 1, num.end()), "[json.exception.invalid_iterator.204] iterators out of range"); + CHECK_MSG(arr.erase(static_cast(10)), "[json.exception.out_of_range.401] array index 10 is out of range"); + } + + SECTION("insert") + { + CHECK_MSG(arr.insert(arr2.begin(), 1), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(arr.insert(arr2.begin(), 2, 1), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(arr.insert(arr2.begin(), arr2.begin(), arr2.end()), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(arr.insert(arr.begin(), arr.begin(), arr2.end()), "[json.exception.invalid_iterator.210] iterators do not fit"); + CHECK_MSG(arr.insert(arr.begin(), arr.begin(), arr.end()), "[json.exception.invalid_iterator.211] passed iterators may not belong to container"); + CHECK_MSG(arr.insert(arr.begin(), obj.begin(), obj.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to arrays"); + CHECK_MSG(arr.insert(arr2.begin(), {1, 2}), "[json.exception.invalid_iterator.202] iterator does not fit current value"); + CHECK_MSG(obj.insert(obj2.begin(), arr.end()), "[json.exception.invalid_iterator.210] iterators do not fit"); + CHECK_MSG(obj.insert(arr.begin(), arr.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to objects"); + CHECK_MSG(obj.update(arr.begin(), arr2.end()), "[json.exception.invalid_iterator.210] iterators do not fit"); + } + + SECTION("patch") + { + const auto patch_msg = [](const char* doc, const char* patch) + { + try + { + json d = json::parse(doc); + d.patch(json::parse(patch)); + } + catch (const json::exception& e) + { + // with JSON_DIAGNOSTICS, what() names the offending value like "(/0) "; drop it + std::string msg = e.what(); + const auto pos = msg.find(" (/"); + if (pos != std::string::npos) + { + msg.erase(pos + 1, msg.find(") ", pos) - pos + 1); + } + return msg; + } + return std::string(); + }; + + CHECK(patch_msg("{}", "{}") == "[json.exception.parse_error.104] parse error: JSON patch must be an array of objects"); + CHECK(patch_msg("{}", "[1]") == "[json.exception.parse_error.104] parse error: JSON patch must be an array of objects"); + CHECK(patch_msg("{}", "[{}]") == "[json.exception.parse_error.105] parse error: operation must have member 'op'"); + CHECK(patch_msg("{}", "[{\"op\":1}]") == "[json.exception.parse_error.105] parse error: operation must have string member 'op'"); + CHECK(patch_msg("{}", "[{\"op\":\"add\"}]") == "[json.exception.parse_error.105] parse error: operation 'add' must have member 'path'"); + CHECK(patch_msg("{}", "[{\"op\":\"add\",\"path\":1}]") == "[json.exception.parse_error.105] parse error: operation 'add' must have string member 'path'"); + CHECK(patch_msg("{}", "[{\"op\":\"foo\",\"path\":\"\"}]") == "[json.exception.parse_error.105] parse error: operation value 'foo' is invalid"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"remove\",\"path\":\"/b\"}]") == "[json.exception.out_of_range.403] key 'b' not found"); + CHECK(patch_msg("[1]", "[{\"op\":\"add\",\"path\":\"/5\",\"value\":0}]") == "[json.exception.out_of_range.401] array index 5 is out of range"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"add\",\"path\":\"/a/b\",\"value\":0}]") == "[json.exception.out_of_range.411] cannot add value: the JSON Patch 'add' target's parent is of type number, but must be an object or array"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"remove\",\"path\":\"/a/b\"}]") == "[json.exception.out_of_range.413] cannot remove value: the JSON Patch 'remove' target's parent is of type number, but must be an object or array"); + CHECK(patch_msg("{\"a\":{}}", "[{\"op\":\"move\",\"from\":\"/a\",\"path\":\"/a/b\"}]") == "[json.exception.out_of_range.414] cannot move value: 'from' path '/a' is a proper prefix of 'path' '/a/b'"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"test\",\"path\":\"/a\",\"value\":2}]") == "[json.exception.other_error.501] unsuccessful: {\"op\":\"test\",\"path\":\"/a\",\"value\":2}"); + CHECK(patch_msg("{\"a\":1}", "[{\"op\":\"move\",\"path\":\"/a\"}]") == "[json.exception.parse_error.105] parse error: operation 'move' must have member 'from'"); + } +} diff --git a/tests/src/unit-to_chars.cpp b/tests/src/unit-to_chars.cpp index c9b72fc97..95c5f2bd0 100644 --- a/tests/src/unit-to_chars.cpp +++ b/tests/src/unit-to_chars.cpp @@ -21,11 +21,8 @@ using nlohmann::detail::dtoa_impl::reinterpret_bits; #include #include #include -#include #include -#include #include -#include #include #include #include @@ -689,20 +686,13 @@ void check_shortest(double v) const std::string& digits = de.first; if (digits.size() > 1) { - // the decimals of one digit fewer next to the value - // (a stream rather than snprintf("%.*e"), whose output GCC cannot bound) - std::ostringstream shorter; - shorter.imbue(std::locale::classic()); - shorter << std::scientific << std::setprecision(static_cast(digits.size()) - 2) << v; - const auto near = digits_and_exponent(shorter.str()); - // as an integer with digits.size() - 1 digits - std::string m = near.first; - int e = near.second; - while (m.size() < digits.size() - 1) - { - m += '0'; - --e; - } + // the decimals of one digit fewer next to the value: v lies within + // half a unit of the last digit of text, so the closest ones are the + // truncated digits and their successor + // (not a stream with std::setprecision(digits.size() - 2): the + // standard library of MSVC 2017 ignores a precision of 0) + const std::string m = digits.substr(0, digits.size() - 1); + const int e = de.second + 1; const std::uint64_t mid = std::stoull(m); for (const std::uint64_t candidate : { diff --git a/tests/src/unit-to_json-rvalue.cpp b/tests/src/unit-to_json-rvalue.cpp new file mode 100644 index 000000000..261493f7a --- /dev/null +++ b/tests/src/unit-to_json-rvalue.cpp @@ -0,0 +1,257 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) + #if __has_include() + #include + #define TO_JSON_RVALUE_HAS_FS 1 + #endif +#endif + +namespace +{ +// element type that counts how it is converted to JSON: a conversion from an +// lvalue copies, a conversion from an rvalue moves +struct tracked +{ + std::string payload; + bool moved_from = false; + + explicit tracked(std::string p) : payload(std::move(p)) {} + tracked(const tracked&) = default; + tracked& operator=(const tracked&) = default; + tracked(tracked&& other) noexcept : payload(std::move(other.payload)), moved_from(other.moved_from) + { + other.moved_from = true; + } + tracked& operator=(tracked&& other) noexcept + { + payload = std::move(other.payload); + moved_from = other.moved_from; + other.moved_from = true; + return *this; + } + ~tracked() = default; +}; + +int tracked_copies = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables,misc-use-internal-linkage) +int tracked_moves = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables,misc-use-internal-linkage) + +void to_json(json& j, const tracked& t) +{ + ++tracked_copies; + j = t.payload; +} + +void to_json(json& j, tracked&& t) +{ + ++tracked_moves; + j = std::move(t.payload); + t.moved_from = true; +} + +// user type with its own rvalue-aware serializer +struct udt_container +{ + std::vector values; +}; +} // namespace + +namespace nlohmann +{ +template<> +struct adl_serializer +{ + static void to_json(json& j, const udt_container& c) + { + j = c.values; + j.push_back("copied"); + } + static void to_json(json& j, udt_container&& c) + { + j = std::move(c.values); + j.push_back("moved"); + } +}; +} // namespace nlohmann + +TEST_CASE("to_json from rvalue containers") +{ + const std::string long_a(64, 'a'); + const std::string long_b(64, 'b'); + + SECTION("vector: elements are moved") + { + std::vector v{long_a, long_b}; + const json j = std::move(v); + CHECK(j == json::array({long_a, long_b})); + for (const auto& s : v) // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved) + { + CHECK(s.empty()); // long strings leave the source element empty when moved + } + } + + SECTION("vector: lvalues are copied and left untouched") + { + const std::vector v{long_a, long_b}; + const json j = v; + CHECK(j == json::array({long_a, long_b})); + CHECK(v[0] == long_a); + CHECK(v[1] == long_b); + + std::vector v2{long_a, long_b}; + const json j2 = v2; + CHECK(j2 == j); + CHECK(v2[0] == long_a); + CHECK(v2[1] == long_b); + } + + SECTION("tracked elements: rvalue moves, lvalue copies") + { + tracked_copies = 0; + tracked_moves = 0; + + std::vector v; + v.emplace_back("x"); + v.emplace_back("y"); + + const json copy = v; + CHECK(copy == json::array({"x", "y"})); + CHECK(tracked_copies == 2); + CHECK(tracked_moves == 0); + CHECK_FALSE(v[0].moved_from); + + const json moved = std::move(v); + CHECK(moved == copy); + CHECK(tracked_copies == 2); + CHECK(tracked_moves == 2); + } + + SECTION("const rvalue is copied") + { + tracked_copies = 0; + tracked_moves = 0; + const std::vector v(2, tracked("z")); + const json j = std::move(v); // NOLINT(performance-move-const-arg,hicpp-move-const-arg) + CHECK(j == json::array({"z", "z"})); + CHECK(tracked_copies == 2); + CHECK(tracked_moves == 0); + } + + SECTION("other sequence containers") + { + CHECK(json(std::deque {long_a, long_b}) == json::array({long_a, long_b})); + CHECK(json(std::list {long_a, long_b}) == json::array({long_a, long_b})); + CHECK(json(std::forward_list {long_a, long_b}) == json::array({long_a, long_b})); + CHECK(json(std::array {{long_a, long_b}}) == json::array({long_a, long_b})); + CHECK(json(std::set {long_a, long_b}) == json::array({long_a, long_b})); + CHECK(json(std::vector {true, false}) == json::array({true, false})); + CHECK(json(std::vector {1, 2, 3}) == json::array({1, 2, 3})); + CHECK(json(std::vector> {{long_a}, {long_b}}) == json::array({json::array({long_a}), json::array({long_b})})); + CHECK(json(std::vector {1, "a"}) == json::array({1, "a"})); + } + + SECTION("valarray") + { + std::valarray va(3); + va[0] = 1; + va[1] = 2; + va[2] = 3; + CHECK(json(std::move(va)) == json::array({1, 2, 3})); // NOLINT(performance-move-const-arg,hicpp-move-const-arg) + CHECK(json(std::valarray(2)) == json::array({0, 0})); + } + + SECTION("array_t and json arrays") + { + json::array_t arr{1, 2}; + CHECK(json(std::move(arr)) == json::array({1, 2})); + json a = json::array({1, 2}); + const json b = std::move(a); + CHECK(b == json::array({1, 2})); + } + + SECTION("object-like containers") + { + std::map m{{"a", long_a}, {"b", long_b}}; + const json j = std::move(m); + CHECK(j == json({{"a", long_a}, {"b", long_b}})); + for (const auto& kv : m) // NOLINT(bugprone-use-after-move,hicpp-invalid-access-moved) + { + CHECK(kv.second.empty()); // values moved, keys (const) copied + } + + const std::map cm{{"a", long_a}}; + const json jc = cm; + CHECK(jc["a"] == long_a); + CHECK(cm.at("a") == long_a); + + std::unordered_map um{{"a", long_a}}; + CHECK(json(std::move(um)) == json({{"a", long_a}})); + + std::multimap mm{{"a", 1}}; + CHECK(json(std::move(mm)) == json({{"a", 1}})); + + tracked_copies = 0; + tracked_moves = 0; + std::map mt; + mt.emplace("k", tracked("v")); + const json jt = std::move(mt); + CHECK(jt == json({{"k", "v"}})); + CHECK(tracked_moves == 1); + CHECK(tracked_copies == 0); + + json::object_t obj{{"a", 1}}; + CHECK(json(std::move(obj)) == json({{"a", 1}})); + } + + SECTION("user type with its own to_json overloads") + { + udt_container c{{1, 2}}; + const json lv = c; + CHECK(lv == json::array({1, 2, "copied"})); + const json rv = std::move(c); + CHECK(rv == json::array({1, 2, "moved"})); + } + + SECTION("assignment and implicit conversion") + { + std::vector v{long_a}; + json j; + j = std::move(v); + CHECK(j == json::array({long_a})); + json k = {std::vector{long_b}}; + CHECK(k.size() == 1); + } + +#ifdef TO_JSON_RVALUE_HAS_FS + SECTION("std::filesystem::path rvalue is still a string") + { + CHECK(json(std::filesystem::path("a/b")) == json("a/b")); +#ifdef __cpp_lib_char8_t + CHECK(json(std::u8string(u8"a/b")) == json("a/b")); +#endif + } +#endif +}