diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 90e356b8a..48b8db457 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,14 +38,14 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 with: languages: c-cpp # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/autobuild@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 diff --git a/.github/workflows/comment_check_amalgamation.yml b/.github/workflows/comment_check_amalgamation.yml index c01572a44..fc96d5c18 100644 --- a/.github/workflows/comment_check_amalgamation.yml +++ b/.github/workflows/comment_check_amalgamation.yml @@ -37,6 +37,14 @@ jobs: var matchArtifact = artifacts.data.artifacts.filter((artifact) => { return artifact.name == "pr" })[0]; + // a run that failed before its jobs started (e.g. a stale queued + // run cancelled long after its push) has no artifact to report on + if (!matchArtifact) { + core.notice('Run ${{ github.event.workflow_run.id }} has no "pr" artifact; nothing to comment on.'); + core.setOutput('found', 'false'); + return; + } + core.setOutput('found', 'true'); var download = await github.rest.actions.downloadArtifact({ owner: context.repo.owner, repo: context.repo.repo, @@ -51,9 +59,11 @@ jobs: # Extract the untrusted PR artifact into a dedicated empty directory and # read only the two expected files by fixed path afterwards. This avoids a # malicious archive overwriting workspace files or escaping via ../ paths. - - run: unzip -o pr.zip -d ./pr_artifact + - if: steps.download.outputs.found == 'true' + run: unzip -o pr.zip -d ./pr_artifact - name: 'Comment on PR' + if: steps.download.outputs.found == 'true' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/flawfinder.yml b/.github/workflows/flawfinder.yml index 82d365c6d..e8cb9f71f 100644 --- a/.github/workflows/flawfinder.yml +++ b/.github/workflows/flawfinder.yml @@ -47,6 +47,6 @@ jobs: output: 'flawfinder_results.sarif' - name: Upload analysis results to GitHub Security tab - uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 with: sarif_file: ${{github.workspace}}/flawfinder_results.sarif diff --git a/.github/workflows/json_view_benchmarks.yml b/.github/workflows/json_view_benchmarks.yml index d1554b85d..305f1ef35 100644 --- a/.github/workflows/json_view_benchmarks.yml +++ b/.github/workflows/json_view_benchmarks.yml @@ -54,7 +54,7 @@ jobs: timeout-minutes: 60 steps: - name: Harden Runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 with: egress-policy: audit @@ -78,7 +78,7 @@ jobs: - name: Summary run: cat tests/benchmarks/json_view/results/*.md >> "$GITHUB_STEP_SUMMARY" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: json_view-benchmarks-${{ matrix.runner }}-${{ inputs.compiler || 'g++' }} path: tests/benchmarks/json_view/results/ diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 0cf262fca..dbebf3e23 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -80,6 +80,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 with: sarif_file: results.sarif diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index ddf181b17..203a1416f 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -65,7 +65,7 @@ jobs: # Upload SARIF file generated in previous step - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3 + uses: github/codeql-action/upload-sarif@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 with: sarif_file: semgrep.sarif if: always() diff --git a/docs/docset/generate_docset.py b/docs/docset/generate_docset.py index 6f6a6e423..768c20d92 100755 --- a/docs/docset/generate_docset.py +++ b/docs/docset/generate_docset.py @@ -341,13 +341,14 @@ def download(url, docs) -> str: raise ValueError(f'not an http(s) URL: {url}') req = urllib.request.Request(u.geturl(), headers={'User-Agent': USER_AGENT}) # (the scheme is checked above) + # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected with urllib.request.urlopen(req, timeout=20) as r: # nosec B310 data = r.read() ctype = r.headers.get_content_type() path = urllib.parse.unquote(u.path).lstrip('/') ext = os.path.splitext(path)[1] if u.query or not ext or path.endswith('/'): - digest = hashlib.sha1(url.encode(), usedforsecurity=False).hexdigest()[:12] + digest = hashlib.sha256(url.encode()).hexdigest()[:12] path = os.path.join(os.path.dirname(path), digest + CONTENT_TYPE_EXT.get(ctype, ext or '.bin')) rel = os.path.normpath(os.path.join('assets', 'external', u.hostname, path)) out = os.path.join(docs, rel) diff --git a/docs/mkdocs/docs/api/adl_serializer/index.md b/docs/mkdocs/docs/api/adl_serializer/index.md index e83b66dec..e994a30cf 100644 --- a/docs/mkdocs/docs/api/adl_serializer/index.md +++ b/docs/mkdocs/docs/api/adl_serializer/index.md @@ -30,6 +30,14 @@ struct adl_serializer { - [**from_json**](from_json.md) - convert a JSON value to any value type - [**to_json**](to_json.md) - convert any value type to a JSON value +## See also + +- [Arbitrary type conversions](../../features/arbitrary_types.md) - how to make your types convertible +- [**get**](../basic_json/get.md) / [**get_to**](../basic_json/get_to.md) - convert a JSON value to a C++ type +- [**basic_json**](../basic_json/basic_json.md) - construct a JSON value from a C++ value +- [**NLOHMANN_DEFINE_TYPE_INTRUSIVE**](../macros/nlohmann_define_type_intrusive.md) - generate the conversion functions for a type +- [Specializing enum conversion](../../features/enum_conversion.md) + ## Version history - Added in version 2.1.0. diff --git a/docs/mkdocs/docs/api/basic_json/at.md b/docs/mkdocs/docs/api/basic_json/at.md index 22df4b5bb..194b123ce 100644 --- a/docs/mkdocs/docs/api/basic_json/at.md +++ b/docs/mkdocs/docs/api/basic_json/at.md @@ -239,9 +239,8 @@ Strong exception safety: if an exception occurs, the original value stays intact 1. Added in version 1.0.0. 2. Added in version 1.0.0. -3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as - already supported by [`operator[]`](operator[].md), [`value`](value.md), [`find`](find.md), and other lookup - functions. +3. Added in version 3.11.0. Support for keys convertible to `std::string_view` added in version 3.13.0, consistently + across the lookup functions. 4. Added in version 2.0.0. Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of [`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) for a one-character array index that is not a digit (e.g., `/x`) in version 3.13.0, as it already did for longer ones. diff --git a/docs/mkdocs/docs/api/basic_json/back.md b/docs/mkdocs/docs/api/basic_json/back.md index 685c1c84b..32066d8c7 100644 --- a/docs/mkdocs/docs/api/basic_json/back.md +++ b/docs/mkdocs/docs/api/basic_json/back.md @@ -63,4 +63,4 @@ Constant. ## Version history - Added in version 1.0.0. -- Adjusted code to return reference to binary values in version 3.8.0. +- Binary values are supported since version 3.8.0. diff --git a/docs/mkdocs/docs/api/basic_json/basic_json.md b/docs/mkdocs/docs/api/basic_json/basic_json.md index e553509ea..d3df598ad 100644 --- a/docs/mkdocs/docs/api/basic_json/basic_json.md +++ b/docs/mkdocs/docs/api/basic_json/basic_json.md @@ -482,15 +482,15 @@ basic_json(basic_json&& other) noexcept; ## Version history -1. Since version 1.0.0. -2. Since version 1.0.0. -3. Since version 2.1.0. -4. Since version 3.2.0. Explicit for different string types if `JSON_USE_IMPLICIT_CONVERSIONS` is `0` since +1. Added in version 1.0.0. +2. Added in version 1.0.0. +3. Added in version 2.1.0. +4. Added in version 3.1.2. Explicit for different string types if `JSON_USE_IMPLICIT_CONVERSIONS` is `0` since version 3.13.0. -5. Since version 1.0.0. -6. Since version 1.0.0. -7. Since version 1.0.0. Fixed in version 3.13.0 to also check the iterator range for binary values; before, a range +5. Added in version 1.0.0. +6. Added in version 1.0.0. +7. Added in version 1.0.0. Fixed in version 3.13.0 to also check the iterator range for binary values; before, a range that did not cover the whole value (such as `(end(), end())`) was accepted and the whole binary value was copied, unlike the other primitive types. -8. Since version 1.0.0. -9. Since version 1.0.0. +8. Added in version 1.0.0. +9. Added in version 1.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/contains.md b/docs/mkdocs/docs/api/basic_json/contains.md index 64a3e49a1..c6e2e7d77 100644 --- a/docs/mkdocs/docs/api/basic_json/contains.md +++ b/docs/mkdocs/docs/api/basic_json/contains.md @@ -141,9 +141,8 @@ Logarithmic in the size of the JSON object. ## Version history 1. Added in version 3.11.0. -2. Added in version 3.6.0. Extended template `KeyType` to support comparable types in version 3.11.0. Fixed in - version 3.13.0 to consistently accept `std::string_view`-convertible keys, as already supported by - [`operator[]`](operator[].md), [`at`](at.md), [`value`](value.md), and other lookup functions. +2. Added in version 3.6.0. Extended template `KeyType` to support comparable types in version 3.11.0. Support for keys + convertible to `std::string_view` added in version 3.13.0, consistently across the lookup functions. 3. Added in version 3.7.0. 4. Deleted overloads for integral key types added in version 3.13.0 to reject such calls at compile time instead of causing undefined behavior at runtime. diff --git a/docs/mkdocs/docs/api/basic_json/count.md b/docs/mkdocs/docs/api/basic_json/count.md index 52dc0ff14..580b79884 100644 --- a/docs/mkdocs/docs/api/basic_json/count.md +++ b/docs/mkdocs/docs/api/basic_json/count.md @@ -92,8 +92,7 @@ Logarithmic in the size of the JSON object. ## Version history 1. Added in version 3.11.0. -2. Added in version 1.0.0. Changed parameter `key` type to `KeyType&&` in version 3.11.0. Fixed in version 3.13.0 to - consistently accept `std::string_view`-convertible keys, as already supported by [`operator[]`](operator[].md), - [`at`](at.md), [`value`](value.md), and other lookup functions. +2. Added in version 1.0.0. Changed to support comparable key types in version 3.11.0. Support for keys convertible to + `std::string_view` added in version 3.13.0, consistently across the lookup functions. 3. Deleted overload for integral key types added in version 3.13.0 to reject such calls at compile time instead of causing undefined behavior at runtime. diff --git a/docs/mkdocs/docs/api/basic_json/emplace.md b/docs/mkdocs/docs/api/basic_json/emplace.md index 09953d347..b1fd856ae 100644 --- a/docs/mkdocs/docs/api/basic_json/emplace.md +++ b/docs/mkdocs/docs/api/basic_json/emplace.md @@ -69,6 +69,6 @@ Logarithmic in the size of the container, O(log(`size()`)). ## Version history -- Since version 2.0.8. +- Added in version 2.0.8. - Fixed in version 3.13.0: for [`ordered_json`](../ordered_json.md), the value could previously only be passed as an rvalue; it can now also be passed as an lvalue or a `#!cpp const` lvalue, matching the behavior of `json`. diff --git a/docs/mkdocs/docs/api/basic_json/emplace_back.md b/docs/mkdocs/docs/api/basic_json/emplace_back.md index 782b61f36..38a6d5c3d 100644 --- a/docs/mkdocs/docs/api/basic_json/emplace_back.md +++ b/docs/mkdocs/docs/api/basic_json/emplace_back.md @@ -67,5 +67,5 @@ Amortized constant. ## Version history -- Since version 2.0.8. +- Added in version 2.0.8. - Returns reference since 3.7.0. diff --git a/docs/mkdocs/docs/api/basic_json/erase.md b/docs/mkdocs/docs/api/basic_json/erase.md index 47531fed8..1782b6162 100644 --- a/docs/mkdocs/docs/api/basic_json/erase.md +++ b/docs/mkdocs/docs/api/basic_json/erase.md @@ -213,7 +213,6 @@ Strong exception safety: if an exception occurs, the original value stays intact 1. Added in version 1.0.0. Added support for binary types in version 3.8.0. 2. Added in version 1.0.0. Added support for binary types in version 3.8.0. 3. Added in version 1.0.0. -4. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as - already supported by [`operator[]`](operator[].md), [`at`](at.md), [`value`](value.md), and other lookup - functions. +4. Added in version 3.11.0. Support for keys convertible to `std::string_view` added in version 3.13.0, consistently + across the lookup functions. 5. Added in version 1.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/exception.md b/docs/mkdocs/docs/api/basic_json/exception.md index b592d62ee..88cf51b1f 100644 --- a/docs/mkdocs/docs/api/basic_json/exception.md +++ b/docs/mkdocs/docs/api/basic_json/exception.md @@ -84,4 +84,4 @@ constructor. ## Version history -- Since version 3.0.0. +- Added in version 3.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/find.md b/docs/mkdocs/docs/api/basic_json/find.md index 92ab380c1..8883dc398 100644 --- a/docs/mkdocs/docs/api/basic_json/find.md +++ b/docs/mkdocs/docs/api/basic_json/find.md @@ -99,8 +99,7 @@ Logarithmic in the size of the JSON object. ## Version history 1. Added in version 3.11.0. -2. Added in version 1.0.0. Changed to support comparable types in version 3.11.0. Fixed in version 3.13.0 to - consistently accept `std::string_view`-convertible keys, as already supported by [`operator[]`](operator[].md), - [`at`](at.md), [`value`](value.md), and other lookup functions. +2. Added in version 1.0.0. Changed to support comparable types in version 3.11.0. Support for keys convertible to + `std::string_view` added in version 3.13.0, consistently across the lookup functions. 3. Deleted overloads for integral key types added in version 3.13.0 to reject such calls at compile time instead of causing undefined behavior at runtime. diff --git a/docs/mkdocs/docs/api/basic_json/front.md b/docs/mkdocs/docs/api/basic_json/front.md index dbe92f589..08c871b9a 100644 --- a/docs/mkdocs/docs/api/basic_json/front.md +++ b/docs/mkdocs/docs/api/basic_json/front.md @@ -57,4 +57,4 @@ Constant. ## Version history - Added in version 1.0.0. -- Adjusted code to return reference to binary values in version 3.8.0. +- Binary values are supported since version 3.8.0. diff --git a/docs/mkdocs/docs/api/basic_json/get.md b/docs/mkdocs/docs/api/basic_json/get.md index fef1be766..fcb5c595a 100644 --- a/docs/mkdocs/docs/api/basic_json/get.md +++ b/docs/mkdocs/docs/api/basic_json/get.md @@ -188,6 +188,6 @@ overload (3). ## Version history -1. Since version 2.1.0. -2. Since version 2.1.0. Extended to work with other specializations of `basic_json` in version 3.2.0. -3. Since version 1.0.0. +1. Added in version 2.1.0. +2. Added in version 2.1.0. Extended to work with other specializations of `basic_json` in version 3.1.2. +3. Added in version 1.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/get_to.md b/docs/mkdocs/docs/api/basic_json/get_to.md index 29a78f064..233d151b0 100644 --- a/docs/mkdocs/docs/api/basic_json/get_to.md +++ b/docs/mkdocs/docs/api/basic_json/get_to.md @@ -76,5 +76,5 @@ Depends on the `json_serializer::from_json()` implementation. ## Version history -- Since version 3.3.0. +- Added in version 3.3.0. - Added a `static_assert` with a clear message for `const` arguments in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/invalid_iterator.md b/docs/mkdocs/docs/api/basic_json/invalid_iterator.md index 6c19a3e7e..d757825d8 100644 --- a/docs/mkdocs/docs/api/basic_json/invalid_iterator.md +++ b/docs/mkdocs/docs/api/basic_json/invalid_iterator.md @@ -75,4 +75,4 @@ classDiagram ## Version history -- Since version 3.0.0. +- Added in version 3.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/items.md b/docs/mkdocs/docs/api/basic_json/items.md index 06e725ddd..025002e07 100644 --- a/docs/mkdocs/docs/api/basic_json/items.md +++ b/docs/mkdocs/docs/api/basic_json/items.md @@ -104,8 +104,8 @@ When iterating over an array, `key()` will return the index of the element as st ## Version history -- Added `iterator_wrapper` in version 3.0.0. -- Added `items` and deprecated `iterator_wrapper` in version 3.1.0. +- Added in version 3.1.0 as the replacement of the static function `iterator_wrapper`, which was available since + version 1.0.0 and is deprecated since version 3.1.0. - Added structured binding support in version 3.5.0. !!! warning "Deprecation" diff --git a/docs/mkdocs/docs/api/basic_json/json_serializer.md b/docs/mkdocs/docs/api/basic_json/json_serializer.md index 20e9cfd44..011f7d113 100644 --- a/docs/mkdocs/docs/api/basic_json/json_serializer.md +++ b/docs/mkdocs/docs/api/basic_json/json_serializer.md @@ -51,4 +51,4 @@ deserializes. See [Template Parameter Requirements](../../features/types/templat ## Version history -- Since version 2.0.0. +- Added in version 2.1.0. diff --git a/docs/mkdocs/docs/api/basic_json/merge_patch.md b/docs/mkdocs/docs/api/basic_json/merge_patch.md index 00e1bc8e5..1fafb1ece 100644 --- a/docs/mkdocs/docs/api/basic_json/merge_patch.md +++ b/docs/mkdocs/docs/api/basic_json/merge_patch.md @@ -71,6 +71,6 @@ returned by `#!cpp (*this)[key]`); it is read as it was when `merge_patch()` was ## Version history -- Added in version 3.0.0. +- Added in version 3.1.0. - Fixed use of freed or relocated memory when `apply_patch` is `#!cpp *this` or refers to a value contained in `#!cpp *this`, in version 3.13.0. diff --git a/docs/mkdocs/docs/api/basic_json/operator+=.md b/docs/mkdocs/docs/api/basic_json/operator+=.md index a3dc61b64..3aee3f22d 100644 --- a/docs/mkdocs/docs/api/basic_json/operator+=.md +++ b/docs/mkdocs/docs/api/basic_json/operator+=.md @@ -128,6 +128,6 @@ interpreted as `object_t::value_type` or `std::initializer_list`, se ## Version history -1. Since version 1.0.0. -2. Since version 1.0.0. -3. Since version 2.0.0. +1. Added in version 1.0.0. +2. Added in version 1.0.0. +3. Added in version 2.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/operator[].md b/docs/mkdocs/docs/api/basic_json/operator[].md index d8df9b73a..4356969a9 100644 --- a/docs/mkdocs/docs/api/basic_json/operator[].md +++ b/docs/mkdocs/docs/api/basic_json/operator[].md @@ -284,10 +284,10 @@ Strong exception safety: if an exception occurs, the original value stays intact 1. Added in version 1.0.0. Fixed in version 3.13.0 to throw `#!cpp std::length_error` instead of emptying the array and accessing it out of bounds when `idx` equals the maximum value of `size_type`. A missing index in the const version is guarded by a runtime assertion since version 3.13.0. -2. Added in version 1.0.0. Added overloads for `T* key` in version 1.1.0. Removed overloads for `T* key` (replaced by 3) - in version 3.11.0. -3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as - already supported by [`at`](at.md), [`value`](value.md), [`find`](find.md), and other lookup functions. +2. Added in version 1.0.0. Added overloads for `T* key` in version 1.1.0. The `T* key` overloads forward to + overload 3 since version 3.11.0. +3. Added in version 3.11.0. Support for keys convertible to `std::string_view` added in version 3.13.0, consistently + across the lookup functions. 4. Added in version 2.0.0. A missing array index in the const version is guarded by a runtime assertion since version 3.13.0. Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of [`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) for a one-character array index that is not diff --git a/docs/mkdocs/docs/api/basic_json/operator_ValueType.md b/docs/mkdocs/docs/api/basic_json/operator_ValueType.md index 48abfb3b5..dca5ebfd1 100644 --- a/docs/mkdocs/docs/api/basic_json/operator_ValueType.md +++ b/docs/mkdocs/docs/api/basic_json/operator_ValueType.md @@ -89,6 +89,6 @@ Linear in the size of the JSON value. ## Version history -- Since version 1.0.0. +- Added in version 1.0.0. - Macros `JSON_EXPLICIT`/[`JSON_USE_IMPLICIT_CONVERSIONS`](../macros/json_use_implicit_conversions.md) added in version 3.9.0. diff --git a/docs/mkdocs/docs/api/basic_json/other_error.md b/docs/mkdocs/docs/api/basic_json/other_error.md index 7019e75c5..6149cc3fb 100644 --- a/docs/mkdocs/docs/api/basic_json/other_error.md +++ b/docs/mkdocs/docs/api/basic_json/other_error.md @@ -75,4 +75,4 @@ classDiagram ## Version history -- Since version 3.0.0. +- Added in version 3.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/out_of_range.md b/docs/mkdocs/docs/api/basic_json/out_of_range.md index e1e54ae07..be579b35d 100644 --- a/docs/mkdocs/docs/api/basic_json/out_of_range.md +++ b/docs/mkdocs/docs/api/basic_json/out_of_range.md @@ -76,4 +76,4 @@ classDiagram ## Version history -- Since version 3.0.0. +- Added in version 3.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/parse_error.md b/docs/mkdocs/docs/api/basic_json/parse_error.md index c3f49ef13..0daae8de2 100644 --- a/docs/mkdocs/docs/api/basic_json/parse_error.md +++ b/docs/mkdocs/docs/api/basic_json/parse_error.md @@ -84,4 +84,4 @@ or the end of file. This also holds true when reading a byte vector for binary f ## Version history -- Since version 3.0.0. +- Added in version 3.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/push_back.md b/docs/mkdocs/docs/api/basic_json/push_back.md index 386a589a3..f6ed6a426 100644 --- a/docs/mkdocs/docs/api/basic_json/push_back.md +++ b/docs/mkdocs/docs/api/basic_json/push_back.md @@ -125,6 +125,6 @@ throws. ## Version history -1. Since version 1.0.0. -2. Since version 1.0.0. -3. Since version 2.0.0. +1. Added in version 1.0.0. +2. Added in version 1.0.0. +3. Added in version 2.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/swap.md b/docs/mkdocs/docs/api/basic_json/swap.md index 4704852f3..4bc36b125 100644 --- a/docs/mkdocs/docs/api/basic_json/swap.md +++ b/docs/mkdocs/docs/api/basic_json/swap.md @@ -182,10 +182,10 @@ Constant. ## Version history -1. Since version 1.0.0. Exchanges the `json_base_class_t` subobject along with the value since version 3.13.0. -2. Since version 1.0.0. Exchanges the `json_base_class_t` subobject along with the value since version 3.13.0. -3. Since version 1.0.0. -4. Since version 1.0.0. -5. Since version 1.0.0. -6. Since version 3.8.0. -7. Since version 3.8.0. \ No newline at end of file +1. Added in version 1.0.0. Exchanges the `json_base_class_t` subobject along with the value since version 3.13.0. +2. Added in version 1.0.0. Exchanges the `json_base_class_t` subobject along with the value since version 3.13.0. +3. Added in version 1.0.0. +4. Added in version 1.0.0. +5. Added in version 1.0.0. +6. Added in version 3.8.0. +7. Added in version 3.8.0. \ No newline at end of file diff --git a/docs/mkdocs/docs/api/basic_json/to_bjdata.md b/docs/mkdocs/docs/api/basic_json/to_bjdata.md index aa0240a73..a85d8ff8a 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/to_bjdata.md @@ -124,5 +124,3 @@ Linear in the size of the JSON value `j`. [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an array or object was silently skipped, producing invalid BJData. -- Writes unsigned integers wider than 64 bits as high-precision numbers since version 3.13.0; previously, they were - silently truncated to 64 bits. \ No newline at end of file diff --git a/docs/mkdocs/docs/api/basic_json/type_error.md b/docs/mkdocs/docs/api/basic_json/type_error.md index b13fdd278..dcc30b29a 100644 --- a/docs/mkdocs/docs/api/basic_json/type_error.md +++ b/docs/mkdocs/docs/api/basic_json/type_error.md @@ -76,4 +76,4 @@ classDiagram ## Version history -- Since version 3.0.0. +- Added in version 3.0.0. diff --git a/docs/mkdocs/docs/api/basic_json/value.md b/docs/mkdocs/docs/api/basic_json/value.md index 12293fd1e..aee09a537 100644 --- a/docs/mkdocs/docs/api/basic_json/value.md +++ b/docs/mkdocs/docs/api/basic_json/value.md @@ -231,11 +231,8 @@ changes to any JSON value. 1. Added in version 1.0.0. Changed parameter `default_value` type from `const ValueType&` to `ValueType&&` in version 3.11.0. Deleted overload for integral key types added in version 3.13.0 to reject such calls at compile time instead of causing undefined behavior at runtime. -2. Added in version 3.11.0. Made `ValueType` the first template parameter in version 3.11.2. Fixed in version 3.13.0 - to consistently accept `std::string_view`-convertible keys, as already supported by - [`operator[]`](operator[].md), [`at`](at.md), [`find`](find.md), and other lookup functions. -3. Added in version 2.0.2. Extended to work with arrays in version 3.13.0, including fixing an issue where resolving - `ptr` through an array unexpectedly threw `out_of_range` instead of returning the resolved element (or - `default_value`, as documented). Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) +2. Added in version 3.11.0. Made `ValueType` the first template parameter in version 3.11.2. Support for keys + convertible to `std::string_view` added in version 3.13.0, consistently across the lookup functions. +3. Added in version 2.0.2. Extended to work with arrays in version 3.13.0. Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of returning `default_value` for a one-character array index that is not a digit (e.g., `/x`) in version 3.13.0, as it already did for longer ones. diff --git a/docs/mkdocs/docs/api/basic_json_document/index.md b/docs/mkdocs/docs/api/basic_json_document/index.md index 69716b779..ea5f449db 100644 --- a/docs/mkdocs/docs/api/basic_json_document/index.md +++ b/docs/mkdocs/docs/api/basic_json_document/index.md @@ -117,9 +117,10 @@ points apply to every edit: ## See also -- [basic_json_view](../basic_json_view/index.md) - the read-only handle to a value of a document -- [json_document](../json_document.md) - this template for the default specialization `nlohmann::json` -- [Zero-copy JSON views](../../features/json_view.md) - the concept behind documents and views +- [Zero-copy JSON views](../../features/json_view.md) - guide to documents and views +- [**basic_json_view**](../basic_json_view/index.md) - the non-owning handle to a value in a document +- [**json_document**](../json_document.md) and [**json_editable_document**](../json_editable_document.md) - the default specializations +- [**materialize**](../basic_json_view/materialize.md) - build a `basic_json` value from a view ## Version history diff --git a/docs/mkdocs/docs/api/basic_json_view/at.md b/docs/mkdocs/docs/api/basic_json_view/at.md index ce142c23f..afbf81620 100644 --- a/docs/mkdocs/docs/api/basic_json_view/at.md +++ b/docs/mkdocs/docs/api/basic_json_view/at.md @@ -80,7 +80,8 @@ None of these exceptions carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics Objects with 128 or more members get a hash index of their keys while parsing, so that a lookup in them takes constant time on average. 2. Linear in `idx`: elements are skipped one at a time from the first one, since they are not a fixed size in the - index (unlike `BasicJsonType`'s array, which is random-access). + index (unlike `BasicJsonType`'s array, which is random-access). Constant if no element of the array is an array + or object, since the elements are then adjacent in the index. 3. Linear in the number of reference tokens of `ptr` and, for each token, in the number of members of the object at that level (as 1.) or the index into the array (as 2.). diff --git a/docs/mkdocs/docs/api/basic_json_view/back.md b/docs/mkdocs/docs/api/basic_json_view/back.md index 1d70e9feb..acb6ebd65 100644 --- a/docs/mkdocs/docs/api/basic_json_view/back.md +++ b/docs/mkdocs/docs/api/basic_json_view/back.md @@ -28,7 +28,8 @@ This exception does not carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics. Linear in the size of the array or object: unlike [`front()`](front.md), which only ever looks at the first element, `back()` has to walk every element to find where they end, since elements are not a fixed size in the -index. +index. Constant if no element of the array is an array or object, since the elements are then adjacent +in the index. ## Notes diff --git a/docs/mkdocs/docs/api/basic_json_view/index.md b/docs/mkdocs/docs/api/basic_json_view/index.md index 69913d650..5f0d90694 100644 --- a/docs/mkdocs/docs/api/basic_json_view/index.md +++ b/docs/mkdocs/docs/api/basic_json_view/index.md @@ -145,9 +145,10 @@ still refers to it -- including ones taken before the change -- reads the new va ## See also -- [basic_json_document](../basic_json_document/index.md) - the document a view refers into -- [materialize](materialize.md) - build the `BasicJsonType` value of a subtree -- [Zero-copy JSON views](../../features/json_view.md) - the concept behind documents and views +- [Zero-copy JSON views](../../features/json_view.md) - guide to documents and views +- [**basic_json_document**](../basic_json_document/index.md) - the document that owns the parsed data +- [**json_view**](../json_view.md) and [**json_editable_view**](../json_editable_view.md) - the default specializations +- [**materialize**](materialize.md) - build a `basic_json` value from a view ## Version history diff --git a/docs/mkdocs/docs/api/basic_json_view/operator[].md b/docs/mkdocs/docs/api/basic_json_view/operator[].md index 354813bdf..8aacf94c9 100644 --- a/docs/mkdocs/docs/api/basic_json_view/operator[].md +++ b/docs/mkdocs/docs/api/basic_json_view/operator[].md @@ -86,7 +86,8 @@ None of these exceptions carry a [`JSON_DIAGNOSTICS`](../macros/json_diagnostics Objects with 128 or more members get a hash index of their keys while parsing, so that a lookup in them takes constant time on average. 2. Linear in `idx`: elements are skipped one at a time from the first one, since they are not a fixed size in the - index (unlike `BasicJsonType`'s array, which is random-access). + index (unlike `BasicJsonType`'s array, which is random-access). Constant if no element of the array is an array + or object, since the elements are then adjacent in the index. 3. Linear in the number of reference tokens of `ptr` and, for each token, in the number of members of the object at that level (as 1.) or the index into the array (as 2.). diff --git a/docs/mkdocs/docs/api/byte_container_with_subtype/byte_container_with_subtype.md b/docs/mkdocs/docs/api/byte_container_with_subtype/byte_container_with_subtype.md index 23d5c9beb..62b0e24fc 100644 --- a/docs/mkdocs/docs/api/byte_container_with_subtype/byte_container_with_subtype.md +++ b/docs/mkdocs/docs/api/byte_container_with_subtype/byte_container_with_subtype.md @@ -51,4 +51,4 @@ byte_container_with_subtype(container_type&& container, subtype_type subtype); ## Version history -Since version 3.8.0. +Added in version 3.8.0. diff --git a/docs/mkdocs/docs/api/byte_container_with_subtype/clear_subtype.md b/docs/mkdocs/docs/api/byte_container_with_subtype/clear_subtype.md index 7fd8fe9a4..ec2e61760 100644 --- a/docs/mkdocs/docs/api/byte_container_with_subtype/clear_subtype.md +++ b/docs/mkdocs/docs/api/byte_container_with_subtype/clear_subtype.md @@ -39,4 +39,4 @@ Constant. ## Version history -Since version 3.8.0. +Added in version 3.8.0. diff --git a/docs/mkdocs/docs/api/byte_container_with_subtype/has_subtype.md b/docs/mkdocs/docs/api/byte_container_with_subtype/has_subtype.md index 98596dab0..e22429f9c 100644 --- a/docs/mkdocs/docs/api/byte_container_with_subtype/has_subtype.md +++ b/docs/mkdocs/docs/api/byte_container_with_subtype/has_subtype.md @@ -42,4 +42,4 @@ Constant. ## Version history -Since version 3.8.0. +Added in version 3.8.0. diff --git a/docs/mkdocs/docs/api/byte_container_with_subtype/index.md b/docs/mkdocs/docs/api/byte_container_with_subtype/index.md index cb5a3781d..b184d8496 100644 --- a/docs/mkdocs/docs/api/byte_container_with_subtype/index.md +++ b/docs/mkdocs/docs/api/byte_container_with_subtype/index.md @@ -29,6 +29,14 @@ specific naming scheme in order to override the binary type. - [**has_subtype**](has_subtype.md) - return whether the value has a subtype - [**clear_subtype**](clear_subtype.md) - clears the binary subtype +## See also + +- [Binary values](../../features/binary_values.md) - how binary values and subtypes are handled +- [**binary**](../basic_json/binary.md) - create a binary array value +- [**get_binary**](../basic_json/get_binary.md) - get a reference to the stored binary value +- [**binary_t**](../basic_json/binary_t.md) - type for binary arrays +- [Binary formats](../../features/binary_formats/index.md) - formats that can store binary values + ## Version history - Added in version 3.8.0. diff --git a/docs/mkdocs/docs/api/byte_container_with_subtype/set_subtype.md b/docs/mkdocs/docs/api/byte_container_with_subtype/set_subtype.md index 6cd319e1d..281de5aef 100644 --- a/docs/mkdocs/docs/api/byte_container_with_subtype/set_subtype.md +++ b/docs/mkdocs/docs/api/byte_container_with_subtype/set_subtype.md @@ -44,4 +44,4 @@ Constant. ## Version history -Since version 3.8.0. +Added in version 3.8.0. diff --git a/docs/mkdocs/docs/api/json.md b/docs/mkdocs/docs/api/json.md index 55504ef9a..b41b691ac 100644 --- a/docs/mkdocs/docs/api/json.md +++ b/docs/mkdocs/docs/api/json.md @@ -30,4 +30,4 @@ types. ## Version history -Since version 1.0.0. +Added in version 1.0.0. diff --git a/docs/mkdocs/docs/api/json_document.md b/docs/mkdocs/docs/api/json_document.md index ea365badc..cfa22e80b 100644 --- a/docs/mkdocs/docs/api/json_document.md +++ b/docs/mkdocs/docs/api/json_document.md @@ -32,4 +32,4 @@ specialization. ## Version history -Since version 3.13.0. +Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/json_editable_document.md b/docs/mkdocs/docs/api/json_editable_document.md index ab3fc3868..ec5226b78 100644 --- a/docs/mkdocs/docs/api/json_editable_document.md +++ b/docs/mkdocs/docs/api/json_editable_document.md @@ -40,4 +40,4 @@ parsing, without ever rewriting the source text -- see [Edits](basic_json_docume ## Version history -Since version 3.13.0. +Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/json_editable_view.md b/docs/mkdocs/docs/api/json_editable_view.md index f5fa59e57..ddee4e372 100644 --- a/docs/mkdocs/docs/api/json_editable_view.md +++ b/docs/mkdocs/docs/api/json_editable_view.md @@ -38,4 +38,4 @@ still referring to it seeing the change, see [Edits](basic_json_document/index.m ## Version history -Since version 3.13.0. +Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/json_pointer/operator_string_t.md b/docs/mkdocs/docs/api/json_pointer/operator_string_t.md index 9fd8bce94..44dde733d 100644 --- a/docs/mkdocs/docs/api/json_pointer/operator_string_t.md +++ b/docs/mkdocs/docs/api/json_pointer/operator_string_t.md @@ -51,5 +51,5 @@ operator string_t() const ## Version history -- Since version 2.0.0. +- Added in version 2.0.0. - Changed type to `string_t` and deprecated in version 3.11.0. diff --git a/docs/mkdocs/docs/api/json_pointer/to_string.md b/docs/mkdocs/docs/api/json_pointer/to_string.md index 2f700e1db..b994ff80e 100644 --- a/docs/mkdocs/docs/api/json_pointer/to_string.md +++ b/docs/mkdocs/docs/api/json_pointer/to_string.md @@ -49,5 +49,5 @@ ptr == json_pointer(ptr.to_string()); ## Version history -- Since version 2.0.0. +- Added in version 2.0.0. - Changed return type to `string_t` in version 3.11.0. diff --git a/docs/mkdocs/docs/api/json_sax/index.md b/docs/mkdocs/docs/api/json_sax/index.md index 19d0a52ac..fec1e0489 100644 --- a/docs/mkdocs/docs/api/json_sax/index.md +++ b/docs/mkdocs/docs/api/json_sax/index.md @@ -59,6 +59,13 @@ leading `-` sign. - [**start_object**](start_object.md) (_virtual_) - the beginning of an object was read - [**string**](string.md) (_virtual_) - a string value was read +## See also + +- [SAX interface](../../features/parsing/sax_interface.md) - guide to event-based parsing +- [**sax_parse**](../basic_json/sax_parse.md) - generate SAX events from an input +- [**parse**](../basic_json/parse.md) - DOM parsing, with optional [parser callbacks](../../features/parsing/parser_callbacks.md) +- [Parse exceptions](../../features/parsing/parse_exceptions.md) + ## Version history - Added in version 3.2.0. diff --git a/docs/mkdocs/docs/api/json_view.md b/docs/mkdocs/docs/api/json_view.md index 819dfe501..89229ed6f 100644 --- a/docs/mkdocs/docs/api/json_view.md +++ b/docs/mkdocs/docs/api/json_view.md @@ -31,4 +31,4 @@ This type is a [`basic_json_view`](basic_json_view/index.md) of a value of a [`j ## Version history -Since version 3.13.0. +Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/macros/index.md b/docs/mkdocs/docs/api/macros/index.md index 1230cd052..ae11a6084 100644 --- a/docs/mkdocs/docs/api/macros/index.md +++ b/docs/mkdocs/docs/api/macros/index.md @@ -137,4 +137,13 @@ header. See also the [macro overview page](../../features/macros.md). - [**NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT_WITH_NAMES**](nlohmann_define_type_with_names.md) - serialize/deserialize a derived class; uses default values; uses custom names - [**NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE_WITH_NAMES**](nlohmann_define_type_with_names.md) - serialize a derived - class; uses custom names \ No newline at end of file + class; uses custom names + +## See also + +- [Macros](../../features/macros.md) - guide to the supported macros +- [Runtime assertions](../../features/assertions.md) +- [Arbitrary type conversions](../../features/arbitrary_types.md) - using the `NLOHMANN_DEFINE_TYPE_*` macros +- [Enum conversion](../../features/enum_conversion.md) - using `NLOHMANN_JSON_SERIALIZE_ENUM` +- [The `nlohmann` namespace](../../features/namespace.md) +- [CMake integration](../../integration/cmake.md) - setting macros from the build system diff --git a/docs/mkdocs/docs/api/ordered_json.md b/docs/mkdocs/docs/api/ordered_json.md index feea642cd..30f295fed 100644 --- a/docs/mkdocs/docs/api/ordered_json.md +++ b/docs/mkdocs/docs/api/ordered_json.md @@ -42,4 +42,4 @@ parsing an object of `n` keys costs O(n²) rather than O(n log n). See ## Version history -Since version 3.9.0. +Added in version 3.9.0. diff --git a/docs/mkdocs/docs/api/ordered_json_document.md b/docs/mkdocs/docs/api/ordered_json_document.md index dd6760ead..b737c2e4d 100644 --- a/docs/mkdocs/docs/api/ordered_json_document.md +++ b/docs/mkdocs/docs/api/ordered_json_document.md @@ -35,4 +35,4 @@ of object keys, instead of sorting them like [`json_document`](json_document.md) ## Version history -Since version 3.13.0. +Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/ordered_json_editable_document.md b/docs/mkdocs/docs/api/ordered_json_editable_document.md index 3127eea70..cf86307e5 100644 --- a/docs/mkdocs/docs/api/ordered_json_editable_document.md +++ b/docs/mkdocs/docs/api/ordered_json_editable_document.md @@ -44,4 +44,4 @@ members [`set`](basic_json_document/set.md) added -- instead of sorting them lik ## Version history -Since version 3.13.0. +Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/ordered_json_editable_view.md b/docs/mkdocs/docs/api/ordered_json_editable_view.md index 78351121e..1c9953d57 100644 --- a/docs/mkdocs/docs/api/ordered_json_editable_view.md +++ b/docs/mkdocs/docs/api/ordered_json_editable_view.md @@ -37,4 +37,4 @@ This type is a [`basic_json_view`](basic_json_view/index.md) of a value of an ## Version history -Since version 3.13.0. +Added in version 3.13.0. diff --git a/docs/mkdocs/docs/api/ordered_json_view.md b/docs/mkdocs/docs/api/ordered_json_view.md index 9800b0752..9df72040f 100644 --- a/docs/mkdocs/docs/api/ordered_json_view.md +++ b/docs/mkdocs/docs/api/ordered_json_view.md @@ -32,4 +32,4 @@ This type is a [`basic_json_view`](basic_json_view/index.md) of a value of an ## Version history -Since version 3.13.0. +Added in version 3.13.0. diff --git a/docs/mkdocs/docs/community/roadmap.md b/docs/mkdocs/docs/community/roadmap.md index 1b1f287a9..d294bd5f7 100644 --- a/docs/mkdocs/docs/community/roadmap.md +++ b/docs/mkdocs/docs/community/roadmap.md @@ -58,7 +58,7 @@ particular, a 3.x release does not: - remove or rename a documented macro, CMake option, CMake target, or header, or change what a documented macro does. Exceptions to these rules, for instance when fixing a bug requires changing the exception a function throws, are -documented in the [release notes](../home/releases.md). +documented in the [release notes](../home/releases/index.md). The following are **not** part of the public API and may change in any release, including patch releases: diff --git a/docs/mkdocs/docs/css/custom.css b/docs/mkdocs/docs/css/custom.css index 1fec0e57c..133609ec9 100644 --- a/docs/mkdocs/docs/css/custom.css +++ b/docs/mkdocs/docs/css/custom.css @@ -18,3 +18,32 @@ white-space: nowrap; cursor: help; } + +/* byte layout tables of the binary formats */ +.md-typeset .byte-layout td code { + white-space: pre-wrap; +} + +.md-typeset .bl { + display: inline-block; + padding: 0 .25em; + border-radius: .2em; + font-family: var(--md-code-font-family); + font-size: .85em; + white-space: nowrap; +} + +.md-typeset .bl-t { + background-color: rgba(66, 133, 244, .25); + box-shadow: inset 0 -.12em 0 rgba(66, 133, 244, .8); +} + +.md-typeset .bl-l { + background-color: rgba(251, 140, 0, .25); + box-shadow: inset 0 -.12em 0 rgba(251, 140, 0, .8); +} + +.md-typeset .bl-d { + background-color: rgba(67, 160, 71, .25); + box-shadow: inset 0 -.12em 0 rgba(67, 160, 71, .8); +} diff --git a/docs/mkdocs/docs/features/binary_formats/bjdata.md b/docs/mkdocs/docs/features/binary_formats/bjdata.md index 12ef19d59..4591652eb 100644 --- a/docs/mkdocs/docs/features/binary_formats/bjdata.md +++ b/docs/mkdocs/docs/features/binary_formats/bjdata.md @@ -190,6 +190,41 @@ The library uses the following mapping from JSON values types to BJData types ac --8<-- "examples/to_bjdata.output" ``` + + + + +### Byte layout + +Consider the following JSON value (shown formatted for readability): + +```json +{ + "id": 300, + "name": "json", + "pi": 3.14, + "tags": [ + "a", + "b" + ] +} +``` + +`to_bjdata` encodes it to 51 bytes (shown in hexadecimal; the encoding contains no whitespace). The colors mark the role of each byte: type/marker length/count/size data. + +
+ + + + + + + + + + +
JSONBytesMeaning
{7Bobject start ({)
"id": 300,69 02 69 64 49 2C 01key: length 2 as int8 (i), then UTF-8 bytes; value: int16 (I), little-endian
"name": "json",69 04 6E 61 6D 65 53 69 04 6A 73 6F 6Ekey: length 4; value: string (S), length 4 as int8, UTF-8 bytes
"pi": 3.14,69 02 70 69 44 1F 85 EB 51 B8 1E 09 40key: length 2; value: float64 (D), little-endian
"tags": [69 04 74 61 67 73 5Bkey: length 4; value: array start ([)
"a",53 69 01 61string, length 1
"b"53 69 01 62string, length 1
]5Darray end (])
}7Dobject end (})
+ ## Deserialization The library maps BJData types to JSON value types as follows: diff --git a/docs/mkdocs/docs/features/binary_formats/bon8.md b/docs/mkdocs/docs/features/binary_formats/bon8.md index d2d20da19..555942def 100644 --- a/docs/mkdocs/docs/features/binary_formats/bon8.md +++ b/docs/mkdocs/docs/features/binary_formats/bon8.md @@ -105,6 +105,41 @@ serialized as 0x81 0x65 0xFF, but `[1,2,3,4,"e"]` as 0x85 0x91 0x92 0x93 0x94 0x --8<-- "examples/to_bon8.output" ``` + + + + +### Byte layout + +Consider the following JSON value (shown formatted for readability): + +```json +{ + "id": 300, + "name": "json", + "pi": 3.14, + "tags": [ + "a", + "b" + ] +} +``` + +`to_bon8` encodes it to 35 bytes (shown in hexadecimal; the encoding contains no whitespace). The colors mark the role of each byte: type/marker length/count/size data. + +
+ + + + + + + + + + +
JSONBytesMeaning
{8Aobject with 4 members
"id": 300,69 64 C4 04key: UTF-8 bytes; the string ends at the next byte, which cannot continue a UTF-8 character; value: 2-byte positive integer (lead byte, then a byte below 0x80)
"name": "json",6E 61 6D 65 FF 6A 73 6F 6E FFkey: UTF-8 bytes; 0xFF terminates the string as another string follows; value: UTF-8 bytes; terminated as a string key follows
"pi": 3.14,70 69 8F 40 09 1E B8 51 EB 85 1Fkey: UTF-8 bytes; value: binary64 (IEEE 754, big-endian); its marker also ends "pi"
"tags": [74 61 67 73 82key: UTF-8 bytes; value: array with 2 elements; its marker also ends "tags"
"a",61 FFUTF-8 bytes; terminated as a string follows
"b"62 FFUTF-8 bytes; terminated as nothing follows
]no end marker: the element count was given up front
}no end marker: the member count was given up front
+ ## Deserialization The library maps BON8 types to JSON value types as follows: diff --git a/docs/mkdocs/docs/features/binary_formats/bson.md b/docs/mkdocs/docs/features/binary_formats/bson.md index f205cba17..7a9f78d9a 100644 --- a/docs/mkdocs/docs/features/binary_formats/bson.md +++ b/docs/mkdocs/docs/features/binary_formats/bson.md @@ -61,6 +61,41 @@ The library uses the following mapping from JSON values types to BSON types: ``` + + + + +### Byte layout + +Consider the following JSON value (shown formatted for readability): + +```json +{ + "id": 300, + "name": "json", + "pi": 3.14, + "tags": [ + "a", + "b" + ] +} +``` + +`to_bson` encodes it to 69 bytes (shown in hexadecimal; the encoding contains no whitespace). The colors mark the role of each byte: type/marker length/count/size data. + +
+ + + + + + + + + + +
JSONBytesMeaning
{45 00 00 00document size: 69 bytes (int32, little-endian)
"id": 300,10 69 64 00 2C 01 00 00int32 element; key as NUL-terminated string; value (little-endian)
"name": "json",02 6E 61 6D 65 00 05 00 00 00 6A 73 6F 6E 00string element; key; length incl. NUL (int32); NUL-terminated UTF-8 bytes
"pi": 3.14,01 70 69 00 1F 85 EB 51 B8 1E 09 40double element; key; value (little-endian)
"tags": [04 74 61 67 73 00 17 00 00 00array element; key; embedded document of 23 bytes (keys are the indices)
"a",02 30 00 02 00 00 00 61 00string element; key "0"; length 2; a + NUL
"b"02 31 00 02 00 00 00 62 00string element; key "1"; length 2; b + NUL
]00end of the array document
}00end of the document
+ ## Deserialization The library maps BSON record types to JSON value types as follows: diff --git a/docs/mkdocs/docs/features/binary_formats/cbor.md b/docs/mkdocs/docs/features/binary_formats/cbor.md index a0a0ed8d0..20f9230fc 100644 --- a/docs/mkdocs/docs/features/binary_formats/cbor.md +++ b/docs/mkdocs/docs/features/binary_formats/cbor.md @@ -110,6 +110,41 @@ see "binary" cells in the table above. --8<-- "examples/to_cbor.output" ``` + + + + +### Byte layout + +Consider the following JSON value (shown formatted for readability): + +```json +{ + "id": 300, + "name": "json", + "pi": 3.14, + "tags": [ + "a", + "b" + ] +} +``` + +`to_cbor` encodes it to 39 bytes (shown in hexadecimal; the encoding contains no whitespace). The colors mark the role of each byte: type/marker length/count/size data. + +
+ + + + + + + + + + +
JSONBytesMeaning
{A4map with 4 key/value pairs
"id": 300,62 69 64 19 01 2Ckey: text string of length 2, then its UTF-8 bytes; value: unsigned integer, 2 bytes follow (big-endian)
"name": "json",64 6E 61 6D 65 64 6A 73 6F 6Ekey: text string of length 4; value: text string of length 4
"pi": 3.14,62 70 69 FB 40 09 1E B8 51 EB 85 1Fkey: text string of length 2; value: double-precision float (IEEE 754, big-endian)
"tags": [64 74 61 67 73 82key: text string of length 4; value: array with 2 elements
"a",61 61text string of length 1
"b"61 62text string of length 1
]no end marker: the element count was given up front
}no end marker: the member count was given up front
+ ## Deserialization The library maps CBOR types to JSON value types as follows: diff --git a/docs/mkdocs/docs/features/binary_formats/index.md b/docs/mkdocs/docs/features/binary_formats/index.md index 2c45995a6..8024b99cf 100644 --- a/docs/mkdocs/docs/features/binary_formats/index.md +++ b/docs/mkdocs/docs/features/binary_formats/index.md @@ -45,12 +45,19 @@ See [binary values](../binary_values.md) for more information. | BJData | 53.2 % | 91.1 % | 78.1 % | 96.6 % | | BJData (size) | 58.6 % | 92.1 % | 86.7 % | 97.4 % | | BJData (size+type) | 58.6 % | 92.1 % | 86.5 % | 97.4 % | -| BON8 | 50.5 % | 83.8 % | 63.5 % | 87.5 % | +| BON8 | **50.5 %** | **83.8 %** | **63.5 %** | **87.5 %** | | BSON | 85.8 % | 95.2 % | 95.8 % | 106.7 % | -| CBOR | 50.5 % | 86.3 % | 68.4 % | 88.0 % | -| MessagePack | 50.5 % | 86.0 % | 68.5 % | 87.9 % | +| CBOR | **50.5 %** | 86.3 % | 68.4 % | 88.0 % | +| MessagePack | **50.5 %** | 86.0 % | 68.5 % | 87.9 % | | UBJSON | 53.2 % | 91.3 % | 78.2 % | 96.6 % | | UBJSON (size) | 58.6 % | 92.3 % | 86.8 % | 97.4 % | | UBJSON (size+type) | 55.9 % | 92.3 % | 85.0 % | 95.0 % | -Sizes compared to minified JSON value. +Sizes compared to minified JSON value; the smallest size per file is printed in bold. + +## See also + +- [Binary values](../binary_values.md) - binary values and subtypes +- [**to_cbor**](../../api/basic_json/to_cbor.md) / [**from_cbor**](../../api/basic_json/from_cbor.md), [**to_msgpack**](../../api/basic_json/to_msgpack.md) / [**from_msgpack**](../../api/basic_json/from_msgpack.md), [**to_bson**](../../api/basic_json/to_bson.md) / [**from_bson**](../../api/basic_json/from_bson.md), [**to_ubjson**](../../api/basic_json/to_ubjson.md) / [**from_ubjson**](../../api/basic_json/from_ubjson.md), [**to_bjdata**](../../api/basic_json/to_bjdata.md) / [**from_bjdata**](../../api/basic_json/from_bjdata.md), [**to_bon8**](../../api/basic_json/to_bon8.md) / [**from_bon8**](../../api/basic_json/from_bon8.md) - the conversion functions +- [Parsing untrusted input](../parsing/untrusted_input.md) +- [SAX interface](../parsing/sax_interface.md) - decode binary formats without building a value diff --git a/docs/mkdocs/docs/features/binary_formats/messagepack.md b/docs/mkdocs/docs/features/binary_formats/messagepack.md index 24eeab139..3bdd1a1be 100644 --- a/docs/mkdocs/docs/features/binary_formats/messagepack.md +++ b/docs/mkdocs/docs/features/binary_formats/messagepack.md @@ -91,6 +91,41 @@ specification: --8<-- "examples/to_msgpack.output" ``` + + + + +### Byte layout + +Consider the following JSON value (shown formatted for readability): + +```json +{ + "id": 300, + "name": "json", + "pi": 3.14, + "tags": [ + "a", + "b" + ] +} +``` + +`to_msgpack` encodes it to 39 bytes (shown in hexadecimal; the encoding contains no whitespace). The colors mark the role of each byte: type/marker length/count/size data. + +
+ + + + + + + + + + +
JSONBytesMeaning
{84map with 4 key/value pairs
"id": 300,A2 69 64 CD 01 2Ckey: fixstr of length 2, then its UTF-8 bytes; value: uint 16 (big-endian)
"name": "json",A4 6E 61 6D 65 A4 6A 73 6F 6Ekey: fixstr of length 4; value: fixstr of length 4
"pi": 3.14,A2 70 69 CB 40 09 1E B8 51 EB 85 1Fkey: fixstr of length 2; value: float 64 (IEEE 754, big-endian)
"tags": [A4 74 61 67 73 92key: fixstr of length 4; value: fixarray with 2 elements
"a",A1 61fixstr of length 1
"b"A1 62fixstr of length 1
]no end marker: the element count was given up front
}no end marker: the member count was given up front
+ ## Deserialization The library maps MessagePack types to JSON value types as follows: diff --git a/docs/mkdocs/docs/features/binary_formats/ubjson.md b/docs/mkdocs/docs/features/binary_formats/ubjson.md index f7a14d855..6b7973627 100644 --- a/docs/mkdocs/docs/features/binary_formats/ubjson.md +++ b/docs/mkdocs/docs/features/binary_formats/ubjson.md @@ -103,6 +103,41 @@ The library uses the following mapping from JSON values types to UBJSON types ac --8<-- "examples/to_ubjson.output" ``` + + + + +### Byte layout + +Consider the following JSON value (shown formatted for readability): + +```json +{ + "id": 300, + "name": "json", + "pi": 3.14, + "tags": [ + "a", + "b" + ] +} +``` + +`to_ubjson` encodes it to 51 bytes (shown in hexadecimal; the encoding contains no whitespace). The colors mark the role of each byte: type/marker length/count/size data. + +
+ + + + + + + + + + +
JSONBytesMeaning
{7Bobject start ({)
"id": 300,69 02 69 64 49 01 2Ckey: length 2 as int8 (i), then UTF-8 bytes; value: int16 (I), big-endian
"name": "json",69 04 6E 61 6D 65 53 69 04 6A 73 6F 6Ekey: length 4; value: string (S), length 4 as int8, UTF-8 bytes
"pi": 3.14,69 02 70 69 44 40 09 1E B8 51 EB 85 1Fkey: length 2; value: float64 (D), big-endian
"tags": [69 04 74 61 67 73 5Bkey: length 4; value: array start ([)
"a",53 69 01 61string, length 1
"b"53 69 01 62string, length 1
]5Darray end (])
}7Dobject end (})
+ ## Deserialization The library maps UBJSON types to JSON value types as follows: diff --git a/docs/mkdocs/docs/features/element_access/index.md b/docs/mkdocs/docs/features/element_access/index.md index 262c057b8..17d0a56db 100644 --- a/docs/mkdocs/docs/features/element_access/index.md +++ b/docs/mkdocs/docs/features/element_access/index.md @@ -21,3 +21,12 @@ flowchart TD A --> F{"just testing first?"} F -->|"yes"| G["contains() / find()"] ``` + +## See also + +- [**operator[]**](../../api/basic_json/operator%5B%5D.md) - unchecked access +- [**at**](../../api/basic_json/at.md) - checked access +- [**value**](../../api/basic_json/value.md) - access with default value +- [**contains**](../../api/basic_json/contains.md) and [**find**](../../api/basic_json/find.md) - test for existence +- [**front**](../../api/basic_json/front.md) and [**back**](../../api/basic_json/back.md) - first and last element +- [**items**](../../api/basic_json/items.md) - iterate over keys and values diff --git a/docs/mkdocs/docs/features/index.md b/docs/mkdocs/docs/features/index.md index 4ffe7e585..88443c796 100644 --- a/docs/mkdocs/docs/features/index.md +++ b/docs/mkdocs/docs/features/index.md @@ -59,3 +59,11 @@ C++ types, and finally serialize it again. This section gives conceptual overviews. For the precise signature, parameters, and return value of a function, see the [API Documentation](../api/basic_json/index.md). + +## See also + +- [API documentation](../api/basic_json/index.md) - reference of all classes and functions +- [FAQ](../home/faq.md) - answers to common questions +- [Exceptions](../home/exceptions.md) - all exceptions and their error codes +- [Design goals](../home/design_goals.md) +- [Integration](../integration/index.md) - how to add the library to a project diff --git a/docs/mkdocs/docs/features/types/index.md b/docs/mkdocs/docs/features/types/index.md index 67e97df6e..03e8c0bae 100644 --- a/docs/mkdocs/docs/features/types/index.md +++ b/docs/mkdocs/docs/features/types/index.md @@ -286,3 +286,12 @@ This implementation does exactly follow this approach, as it uses double precisi ### Storage Integer number values, unsigned integer number values, and floating-point number values are stored directly inside a `basic_json` type. + +## See also + +- [Number handling](number_handling.md) - how numbers are stored and converted +- [Template parameter requirements](template_parameters.md) +- [**basic_json**](../../api/basic_json/index.md) - the class template and its member types +- [**type**](../../api/basic_json/type.md) and [**value_t**](../../api/basic_json/value_t.md) - query the type of a value +- [**object_t**](../../api/basic_json/object_t.md), [**array_t**](../../api/basic_json/array_t.md), [**string_t**](../../api/basic_json/string_t.md), [**number_integer_t**](../../api/basic_json/number_integer_t.md), [**number_float_t**](../../api/basic_json/number_float_t.md) - the underlying types +- [Object order](../object_order.md) diff --git a/docs/mkdocs/docs/home/architecture.md b/docs/mkdocs/docs/home/architecture.md index 51ce62fda..295ea09ba 100644 --- a/docs/mkdocs/docs/home/architecture.md +++ b/docs/mkdocs/docs/home/architecture.md @@ -339,6 +339,11 @@ The `serializer` (used by [`dump`](../api/basic_json/dump.md) and [`operator<<`] - `output_stream_adapter` writes to a `std::ostream`. - `output_string_adapter` appends to a string. +The `binary_writer` does not call an adapter for every byte. The `to_*` functions that return a `std::vector` write +into the vector through a sink (`output_vector_sink`) that is not virtual. The functions that take an output adapter +write through a sink (`output_adapter_sink`) that collects the output in a buffer of one kibibyte and passes it to the +adapter in blocks, so that the adapter is called (and a stream locked) once per block instead of once per value. + ## Value conversion Values are converted from and to other types with the `JSONSerializer` template parameter. The default, diff --git a/docs/mkdocs/docs/home/releases.md b/docs/mkdocs/docs/home/releases.md deleted file mode 100644 index c0865270e..000000000 --- a/docs/mkdocs/docs/home/releases.md +++ /dev/null @@ -1,453 +0,0 @@ -# Releases - -This page summarizes the notable changes of every release and links to the relevant documentation. -The **complete release notes** — including all changes, the download files, and their checksums — are -published on the [GitHub releases page](https://github.com/nlohmann/json/releases). - -!!! info "Unreleased changes" - - This documentation is built from the `develop` branch and may describe changes that are not part of a release - yet. Their version numbers are followed by an unreleased badge. - -## Unreleased - -- Converting an rvalue container to JSON (`json j = std::move(vec);`, `json j = std::map{...};`) - now **moves** the elements instead of copying them (e.g., `std::vector` with 100,000 strings: 4.7 ms to - 2.3 ms). Two consequences: the source container is left in a valid but unspecified (moved-from) state, and if an - exception is thrown during the conversion, the source may be partially moved-from, so the strong exception guarantee - no longer holds for the source of an rvalue conversion (the JSON value itself is unaffected). Lvalues and `const` - objects are still copied. See [`basic_json`](../api/basic_json/basic_json.md). - -## v3.12.0 (2025-04-11) - -Fixes bugs found in 3.11.3 and adds several features. All changes are backward-compatible. - -- Adds diagnostic byte positions via [`JSON_DIAGNOSTIC_POSITIONS`](../api/macros/json_diagnostic_positions.md), - exposed through the new [`start_pos`](../api/basic_json/start_pos.md) and - [`end_pos`](../api/basic_json/end_pos.md) member functions. -- Makes the [conversion macros](../features/arbitrary_types.md#simplify-your-life-with-macros) - templated (so they also work with [`ordered_json`](../api/ordered_json.md)) and adds - [`NLOHMANN_DEFINE_DERIVED_TYPE`](../api/macros/nlohmann_define_derived_type.md) for derived classes. -- Adds `std::optional` support (C++17) and lets [`patch`](../api/basic_json/patch.md), - [`diff`](../api/basic_json/diff.md), and [`flatten`](../api/basic_json/flatten.md) work with - arbitrary string types. -- Extends the [binary formats](../features/binary_formats/index.md): - [BJData](../features/binary_formats/bjdata.md) draft 3 and unsigned 64-bit integers for - [BSON](../features/binary_formats/bson.md). -- Adds multidimensional C-array conversion and UTF-8 encoded `std::filesystem::path` conversions, and - raises the minimum [CMake](../integration/cmake.md) version to 3.5 and supports CMake 4.0. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.12.0). - -## v3.11.3 (2023-11-28) - -Adds features and fixes bugs found in 3.11.2. All changes are backward-compatible. - -- Adds a [custom base class](../api/basic_json/json_base_class_t.md) as a node customization point. -- Adds serialization-only [conversion macros](../features/macros.md) - (`NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE` and `NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE`) - and a clearer parse error for empty input. -- Adds [Bazel](../integration/package_managers.md#bazel) and - [Swift Package Manager](../integration/package_managers.md#swift-package-manager) build support. -- Fixes custom allocators, a memory leak in [`adl_serializer`](../api/adl_serializer/to_json.md)'s - `to_json`, initializer-list construction when `size_type` is not `int`, and many compiler warnings. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.11.3). - -## v3.11.2 (2022-08-12) - -Fixes bugs found in 3.11.1 and restructures the namespace. All changes are backward-compatible. - -- Fixes the [`value`](../api/basic_json/value.md) function (broken for strings, size types, and - `nullptr` in 3.11.0) and makes `json_fwd.hpp` self-contained. -- Restores using [`json_pointer`](../api/json_pointer/index.md) as a key in associative containers and - comparing it with strings. -- Restructures the inline [namespace](../features/namespace.md) and allows disabling the version - component, and avoids heap allocations in the [BJData](../features/binary_formats/bjdata.md) parser. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.11.2). - -## v3.11.1 (2022-08-01) - -Fixes a regression from 3.11.0. All changes are backward-compatible. - -- Restores the global [user-defined string literals](../api/macros/json_use_global_udls.md) - [`operator""_json`](../api/operator_literal_json.md) and - [`operator""_json_pointer`](../api/operator_literal_json_pointer.md), which 3.11.0 had moved into a - namespace by default. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.11.1). - -## v3.11.0 (2022-08-01) - -One of the largest releases ever. All changes are backward-compatible. - -- Allows `std::string_view` as object keys in [`at`](../api/basic_json/at.md), - [`operator[]`](../api/basic_json/operator%5B%5D.md), [`value`](../api/basic_json/value.md), - [`erase`](../api/basic_json/erase.md), [`find`](../api/basic_json/find.md), - [`contains`](../api/basic_json/contains.md), and [`count`](../api/basic_json/count.md). -- Adds the [BJData](../features/binary_formats/bjdata.md) binary format (the fifth supported format). -- Improves C++20 support, including [`operator<=>`](../api/basic_json/operator_spaceship.md) and - ``-compatible iterators. -- Adds a versioned, ABI-tagged inline [namespace](../features/namespace.md) - ([`NLOHMANN_JSON_NAMESPACE`](../api/macros/nlohmann_json_namespace.md)) and the option to move the - UDLs out of the global namespace ([`JSON_USE_GLOBAL_UDLS`](../api/macros/json_use_global_udls.md)). -- Adds [`patch_inplace`](../api/basic_json/patch_inplace.md), default values for the - [conversion macros](../features/arbitrary_types.md#simplify-your-life-with-macros), and an option to - disable enum serialization ([`JSON_DISABLE_ENUM_SERIALIZATION`](../api/macros/json_disable_enum_serialization.md)). - -This release introduced a UDL regression that was fixed in -[3.11.1](https://github.com/nlohmann/json/releases/tag/v3.11.1). -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.11.0). - -## v3.10.5 (2022-01-03) - -Bug-fix release. All changes are backward-compatible. - -- Guards the `std::filesystem` conversions behind compiler-support checks - ([`JSON_HAS_FILESYSTEM`](../api/macros/json_has_filesystem.md)), which can be set to `0` to disable - them altogether. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.5). - -## v3.10.4 (2021-10-16) - -Fixes regressions introduced in 3.10.0. All changes are backward-compatible. - -- Fixes the `std::filesystem::path` conversion (which could trigger a stack overflow and broke - compilation on Windows). -- Fixes compilation for types with an explicit defaulted constructor and for code relying on the - return values of `std::find` and `std::remove`. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.4). - -## v3.10.3 (2021-10-08) - -Fixes more regressions from 3.10.0. All changes are backward-compatible. - -- Fixes [extended-diagnostics](../api/macros/json_diagnostics.md) assertions triggered by - [`update`](../api/basic_json/update.md) and by inserting into arrays. -- Supports custom allocators when writing binary formats into a `std::vector`, and allows conversion - from types that only provide `begin()`/`end()`. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.3). - -## v3.10.2 (2021-08-26) - -Re-release of 3.10.1, whose Git tag pointed at the wrong commit due to a bug in the release script. -All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.2). - -## v3.10.1 (2021-08-24) - -Fixes a regression from 3.10.0. All changes are backward-compatible. - -- Fixes an [extended-diagnostics](../api/macros/json_diagnostics.md) assertion triggered when used - with [`ordered_json`](../api/ordered_json.md), and hardens the GDB pretty-printer. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.1). - -## v3.10.0 (2021-08-17) - -Feature release. All changes are backward-compatible. - -- Adds [extended diagnostic messages](../api/macros/json_diagnostics.md) - ([`JSON_DIAGNOSTICS`](../api/macros/json_diagnostics.md)) that prepend a JSON pointer to exception - messages to pinpoint the offending value. -- Adds a GDB pretty-printer and a [`cbor_tag_handler_t`](../api/basic_json/cbor_tag_handler_t.md) - `store` option to keep CBOR tags as binary subtypes. -- Supports containers with non-default-constructible types and parsing from `std::byte`. -- Adds [`JSON_NO_IO`](../api/macros/json_no_io.md) to exclude the I/O headers and the - [`JSON_HAS_CPP_*`](../api/macros/json_has_cpp_11.md) macros to override the detected C++ standard. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.0). - -## v3.9.1 (2020-08-06) - -Fixes two regressions from 3.9.0. All changes are backward-compatible. - -- Accepts consecutive [comments](../features/comments.md) and completes the - [`ordered_json`](../api/ordered_json.md) interface (e.g. `ordered_json::parse`). - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.9.1). - -## v3.9.0 (2020-07-27) - -Feature release adding four long-requested features. All changes are backward-compatible. - -- Optional [comment](../features/comments.md) parsing in [`parse`](../api/basic_json/parse.md) via the - `ignore_comments` parameter. -- [`ordered_json`](../api/ordered_json.md) to preserve the [insertion order](../features/object_order.md) - of object keys. -- An option to switch off [implicit conversions](../api/macros/json_use_implicit_conversions.md). -- The [`NLOHMANN_DEFINE_TYPE_*`](../features/arbitrary_types.md#simplify-your-life-with-macros) - convenience macros, plus high-precision-number support for - [UBJSON](../features/binary_formats/ubjson.md) and CBOR tag handling. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.9.0). - -## v3.8.0 (2020-06-14) - -Feature release. All changes are backward-compatible. - -- Introduces a [binary value](../features/binary_values.md) type that is read from and written to - [CBOR](../features/binary_formats/cbor.md), [BSON](../features/binary_formats/bson.md), and - [MessagePack](../features/binary_formats/messagepack.md), and can be shared between formats. -- Generalizes the input adapters to read from any `LegacyInputIterator` container (3–10 % faster - parsing). -- Fixes [`contains`](../api/basic_json/contains.md) for JSON pointers and makes the binary - [`from_cbor`](../api/basic_json/from_cbor.md)/[`from_msgpack`](../api/basic_json/from_msgpack.md)/etc. - functions respect `allow_exceptions`. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.8.0). - -## v3.7.3 (2019-11-17) - -Fixes a regression from 3.7.2 that could yield quadratic complexity in destructor calls. All changes -are backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v3.7.3). - -## v3.7.2 (2019-11-10) - -Fixes a stack overflow for deeply nested input by making the destructor iterative; parsing is now -bounded only by available memory. All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.7.2). - -## v3.7.1 (2019-11-06) - -Bug-fix release. All changes are backward-compatible. - -- Fixes a segmentation fault when serializing the `std::int64_t` minimum value and fixes - [`contains`](../api/basic_json/contains.md) for JSON pointers. -- Allows [`items`](../api/basic_json/items.md) with a custom string type and makes `json_pointer::back` - `const`. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.7.1). - -## v3.7.0 (2019-07-28) - -Convenience features and house-keeping. All changes are backward-compatible. - -- Adds a [`contains`](../api/basic_json/contains.md) overload that checks a JSON pointer without - throwing, a generic `to_string`, and a return value for - [`emplace_back`](../api/basic_json/emplace_back.md). - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.7.0). - -## v3.6.1 (2019-03-20) - -Fixes a regression (GCC 7/8 compilation) and a `` build error introduced in 3.6.0. All -changes are backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v3.6.1). - -## v3.6.0 (2019-03-20) - -Feature release. All changes are backward-compatible. - -- Reworks the [JSON pointer](../features/json_pointer.md) interface (`operator/`, `push_back`, - `parent_pointer`, …). -- Adds a [`contains`](../api/basic_json/contains.md) function to test for an object key and greatly - improves the performance of integer serialization. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.6.0). - -## v3.5.0 (2018-12-22) - -Feature release. All changes are backward-compatible. - -- Adds structured-binding support via the [`items`](../api/basic_json/items.md) function and reading - from `FILE*` in the [`parse`](../api/basic_json/parse.md) function. -- Fixes the `eofbit` handling on input streams and a bug in the BSON SAX parser. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.5.0). - -## v3.4.0 (2018-10-30) - -Feature release. All changes are backward-compatible. - -- Adds [BSON](../features/binary_formats/bson.md) read/write support. -- Adds configurable Unicode error handlers to [`dump`](../api/basic_json/dump.md) (throw, replace with - U+FFFD, or ignore) and the - [`NLOHMANN_JSON_SERIALIZE_ENUM`](../api/macros/nlohmann_json_serialize_enum.md) macro for - [enum conversion](../features/enum_conversion.md). -- Improves parse-error messages with line/column positions and context. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.4.0). - -## v3.3.0 (2018-10-05) - -Feature release. All changes are backward-compatible. - -- Adds GCC 4.8 support, the [`get_to`](../api/basic_json/get_to.md) function, and an overhauled and - documented [CMake](../integration/cmake.md) integration. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.3.0). - -## v3.2.0 (2018-08-20) - -Feature release. All changes are backward-compatible. - -- Adds a [SAX interface](../features/parsing/sax_interface.md) and a non-recursive parser. -- Adds parsing from wide-string types (`std::wstring`, `std::u16string`, `std::u32string`) and - `std::string_view` (C++17), and round-tripping of `std::map`/`std::unordered_map` with non-string - keys. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.2.0). - -## v3.1.2 (2018-03-14) - -Bug-fix release. All changes are backward-compatible. - -- Fixes a memory leak in the parser callback and adds user-defined string-type support to the parser - and serializer. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.1.2). - -## v3.1.1 (2018-02-13) - -Bug-fix release. All changes are backward-compatible. - -- Fixes parsing of indefinite-length CBOR strings, a user-defined conversion to vector types, and - overflow detection for UBJSON containers. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.1.1). - -## v3.1.0 (2018-02-01) - -Feature release. All changes are backward-compatible. - -- Adds [UBJSON](../features/binary_formats/ubjson.md) read/write support and - [JSON Merge Patch](../features/merge_patch.md) via [`merge_patch`](../api/basic_json/merge_patch.md). -- Switches to the Grisu2 algorithm for short, round-trippable floating-point output, and splits the - header into [multiple files](../integration/index.md) with a forward-declaration header. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.1.0). - -## v3.0.1 (2017-12-29) - -Fixes small issues in the [JSON Pointer](../features/json_pointer.md) and -[JSON Patch](../features/json_patch.md) implementations (invalid "copy" targets and non-integer array -indices). All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.0.1). - -## v3.0.0 (2017-12-17) - -First 3.x release — a major release with breaking changes (see the -[migration guide](../integration/migration_guide.md)). - -- Introduces user-defined [exceptions](exceptions.md) (`json::exception` and subtypes, each with an - identifier). -- Adds a non-throwing [`accept`](../api/basic_json/accept.md) function and an `allow_exceptions` flag - for [`parse`](../api/basic_json/parse.md), and an [`update`](../api/basic_json/update.md) function to - merge objects. -- Adds streaming for CBOR and MessagePack and allows storing NaN/infinity. -- Non-UTF-8 strings now throw on serialization, and the iterator category changed to bidirectional. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.0.0). - -## v2.1.1 (2017-02-25) - -Bug-fix release. All changes are backward-compatible. - -- Makes number parsing and serialization locale-independent with correct floating-point - round-tripping; released files are now GPG-signed. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.1.1). - -## v2.1.0 (2017-01-28) - -Feature release. All changes are backward-compatible. - -- Adds conversions from and to [arbitrary user-defined types](../features/arbitrary_types.md) via - `to_json`/`from_json`, the [`meta`](../api/basic_json/meta.md) function, and the option to switch off - exceptions ([`JSON_NOEXCEPTION`](../api/macros/json_noexception.md)). - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.1.0). - -## v2.0.10 (2017-01-02) - -Fixes several security-relevant bugs in the CBOR and MessagePack parsers found by OSS-Fuzz. All -changes are backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.10). - -## v2.0.9 (2016-12-16) - -Adds the [CBOR](../features/binary_formats/cbor.md) and -[MessagePack](../features/binary_formats/messagepack.md) binary formats. All changes are -backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.9). - -## v2.0.8 (2016-12-02) - -Adds the [`emplace`](../api/basic_json/emplace.md) and -[`emplace_back`](../api/basic_json/emplace_back.md) functions and improves parsing and serialization -performance. All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.8). - -## v2.0.7 (2016-11-02) - -Fixes several parser bugs found through the "Parsing JSON is a Minefield" study (short files, encoding -detection, surrogate pairs). All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.7). - -## v2.0.6 (2016-10-15) - -Fixes [`operator[]`](../api/basic_json/operator%5B%5D.md) for [JSON pointers](../features/json_pointer.md) -so that it creates missing values like the other overloads. All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.6). - -## v2.0.5 (2016-09-14) - -Fixes a remaining stream end-of-file detection bug in the parser. All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.5). - -## v2.0.4 (2016-09-11) - -Fixes stream end-of-file detection in the parser. All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.4). - -## v2.0.3 (2016-08-31) - -Generalizes the parser to accept any contiguous sequence of one-byte elements and deprecates the -input-stream constructor in favor of the [`parse`](../api/basic_json/parse.md) function. All changes -are backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.3). - -## v2.0.2 (2016-07-31) - -Overhauls the parser (now rejecting unescaped control characters), tightens the class invariants, and -cleans up the code. All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.2). - -## v2.0.1 (2016-06-28) - -Fixes a performance regression in the [`dump`](../api/basic_json/dump.md) function by adjusting the -stream locale once per serialization. All changes are backward-compatible. -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.1). - -## v2.0.0 (2016-06-24) - -Feature release with a minor (potentially non-backward-compatible) API change from added `noexcept` -and `constexpr` specifiers. - -- Adds [JSON Pointer](../features/json_pointer.md) support in [`at`](../api/basic_json/at.md) and - [`operator[]`](../api/basic_json/operator%5B%5D.md), plus [`flatten`](../api/basic_json/flatten.md) - and [`unflatten`](../api/basic_json/unflatten.md). -- Adds [JSON Patch](../features/json_patch.md) via [`diff`](../api/basic_json/diff.md) and - [`patch`](../api/basic_json/patch.md), unsigned 64-bit integer support, and locale-independent - serialization. - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.0). - -## v1.1.0 (2016-01-24) - -Bug-fix and feature release. All changes are backward-compatible. - -- Improves floating-point round-tripping, adds a `get_ref` accessor for stored values, and introduces - runtime [assertions](../features/assertions.md). - -[Full release notes](https://github.com/nlohmann/json/releases/tag/v1.1.0). - -## v1.0.0 (2015-12-28) - -First official release. [Full release notes](https://github.com/nlohmann/json/releases/tag/v1.0.0). - -## See also - -- [Migration Guide](../integration/migration_guide.md) — how to future-proof your code for the next - major version and replace deprecated functions. diff --git a/docs/mkdocs/docs/home/releases/index.md b/docs/mkdocs/docs/home/releases/index.md new file mode 100644 index 000000000..e0dbca2d5 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/index.md @@ -0,0 +1,155 @@ +# Releases + +This section summarizes the notable changes of every release and links to the relevant documentation. +The **complete release notes** — including all changes, the download files, and their checksums — are +published on the [GitHub releases page](https://github.com/nlohmann/json/releases). + +!!! info "Unreleased changes" + + This documentation is built from the `develop` branch and may describe changes that are not part of a release + yet. Their version numbers are followed by an unreleased badge. + +Every release has its own page with a summary of its notable changes and an overview of the API that was added or +changed in it. The API overview is generated from the "Version history" section of the +[API pages](../../api/basic_json/index.md); it is not a change log, but helps to see whether a feature is available in +the version you use. + +## Unreleased + +| Release | Date | Summary | +|---------|------|---------| +| [Unreleased](unreleased.md) | | Changes on the `develop` branch that are not part of a release yet. | + +## Version 3.12.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.12.0](v3.12.0.md) | 2025-04-11 | Fixes bugs found in 3.11.3 and adds several features. No public API was removed or renamed. | + +## Version 3.11.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.11.3](v3.11.3.md) | 2023-11-28 | Adds features and fixes bugs found in 3.11.2. All changes are backward-compatible. | +| [v3.11.2](v3.11.2.md) | 2022-08-12 | Fixes bugs found in 3.11.1 and restructures the namespace. All changes are backward-compatible. | +| [v3.11.1](v3.11.1.md) | 2022-08-01 | Fixes a regression from 3.11.0. All changes are backward-compatible. | +| [v3.11.0](v3.11.0.md) | 2022-08-01 | One of the largest releases ever. All changes are backward-compatible. | + +## Version 3.10.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.10.5](v3.10.5.md) | 2022-01-03 | Bug-fix release. All changes are backward-compatible. | +| [v3.10.4](v3.10.4.md) | 2021-10-16 | Fixes regressions introduced in 3.10.0. All changes are backward-compatible. | +| [v3.10.3](v3.10.3.md) | 2021-10-08 | Fixes more regressions from 3.10.0. All changes are backward-compatible. | +| [v3.10.2](v3.10.2.md) | 2021-08-26 | Re-release of 3.10.1, whose Git tag pointed at the wrong commit due to a bug in the release script. All changes are backward-compatible. | +| [v3.10.1](v3.10.1.md) | 2021-08-24 | Fixes a regression from 3.10.0. All changes are backward-compatible. | +| [v3.10.0](v3.10.0.md) | 2021-08-17 | Feature release. All changes are backward-compatible. | + +## Version 3.9.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.9.1](v3.9.1.md) | 2020-08-06 | Fixes two regressions from 3.9.0. All changes are backward-compatible. | +| [v3.9.0](v3.9.0.md) | 2020-07-27 | Feature release adding four long-requested features. All changes are backward-compatible. | + +## Version 3.8.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.8.0](v3.8.0.md) | 2020-06-14 | Feature release. All changes are backward-compatible. | + +## Version 3.7.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.7.3](v3.7.3.md) | 2019-11-17 | Fixes a regression from 3.7.2 that could yield quadratic complexity in destructor calls. All changes are backward-compatible. | +| [v3.7.2](v3.7.2.md) | 2019-11-10 | Fixes a stack overflow for deeply nested input by making the destructor iterative; parsing of JSON text is now bounded only by available memory (the binary-format parsers still recurse). All changes are backward-compatible. | +| [v3.7.1](v3.7.1.md) | 2019-11-06 | Bug-fix release. All changes are backward-compatible. | +| [v3.7.0](v3.7.0.md) | 2019-07-28 | Convenience features and house-keeping. All changes are backward-compatible. | + +## Version 3.6.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.6.1](v3.6.1.md) | 2019-03-20 | Fixes a regression (GCC 7/8 compilation) and a `` build error introduced in 3.6.0. All changes are backward-compatible. | +| [v3.6.0](v3.6.0.md) | 2019-03-20 | Feature release. All changes are backward-compatible. | + +## Version 3.5.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.5.0](v3.5.0.md) | 2018-12-22 | Feature release. All changes are backward-compatible. | + +## Version 3.4.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.4.0](v3.4.0.md) | 2018-10-30 | Feature release. All changes are backward-compatible. | + +## Version 3.3.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.3.0](v3.3.0.md) | 2018-10-05 | Feature release. All changes are backward-compatible. | + +## Version 3.2.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.2.0](v3.2.0.md) | 2018-08-20 | Feature release. All changes are backward-compatible. | + +## Version 3.1.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.1.2](v3.1.2.md) | 2018-03-14 | Bug-fix release. All changes are backward-compatible. | +| [v3.1.1](v3.1.1.md) | 2018-02-13 | Bug-fix release. All changes are backward-compatible. | +| [v3.1.0](v3.1.0.md) | 2018-02-01 | Feature release. All changes are backward-compatible. | + +## Version 3.0.x + +| Release | Date | Summary | +|---------|------|---------| +| [v3.0.1](v3.0.1.md) | 2017-12-29 | Fixes small issues in the [JSON Pointer](../../features/json_pointer.md) and [JSON Patch](../../features/json_patch.md) implementations (invalid "copy" targets and non-integer array indices). All changes are backward-compatible. | +| [v3.0.0](v3.0.0.md) | 2017-12-17 | First 3.x release — a major release with breaking changes (see the [migration guide](../../integration/migration_guide.md)). | + +## Version 2.1.x + +| Release | Date | Summary | +|---------|------|---------| +| [v2.1.1](v2.1.1.md) | 2017-02-25 | Bug-fix release. All changes are backward-compatible. | +| [v2.1.0](v2.1.0.md) | 2017-01-28 | Feature release. All changes are backward-compatible. | + +## Version 2.0.x + +| Release | Date | Summary | +|---------|------|---------| +| [v2.0.10](v2.0.10.md) | 2017-01-02 | Fixes several security-relevant bugs in the CBOR and MessagePack parsers found by OSS-Fuzz. All changes are backward-compatible. | +| [v2.0.9](v2.0.9.md) | 2016-12-16 | Adds the [CBOR](../../features/binary_formats/cbor.md) and [MessagePack](../../features/binary_formats/messagepack.md) binary formats. All changes are backward-compatible. | +| [v2.0.8](v2.0.8.md) | 2016-12-02 | Adds the [`emplace`](../../api/basic_json/emplace.md) and [`emplace_back`](../../api/basic_json/emplace_back.md) functions and improves parsing and serialization performance. All changes are backward-compatible. | +| [v2.0.7](v2.0.7.md) | 2016-11-02 | Fixes several parser bugs found through the "Parsing JSON is a Minefield" study (short files, encoding detection, surrogate pairs). All changes are backward-compatible. | +| [v2.0.6](v2.0.6.md) | 2016-10-15 | Fixes [`operator[]`](../../api/basic_json/operator%5B%5D.md) for [JSON pointers](../../features/json_pointer.md) so that it creates missing values like the other overloads. All changes are backward-compatible. | +| [v2.0.5](v2.0.5.md) | 2016-09-14 | Fixes a remaining stream end-of-file detection bug in the parser. All changes are backward-compatible. | +| [v2.0.4](v2.0.4.md) | 2016-09-11 | Fixes stream end-of-file detection in the parser. All changes are backward-compatible. | +| [v2.0.3](v2.0.3.md) | 2016-08-31 | Generalizes the parser to accept any contiguous sequence of one-byte elements and deprecates the input-stream constructor in favor of the [`parse`](../../api/basic_json/parse.md) function. All changes are backward-compatible. | +| [v2.0.2](v2.0.2.md) | 2016-07-31 | Overhauls the parser (now rejecting unescaped control characters), tightens the class invariants, and cleans up the code. All changes are backward-compatible. | +| [v2.0.1](v2.0.1.md) | 2016-06-28 | Fixes a performance regression in the [`dump`](../../api/basic_json/dump.md) function by adjusting the stream locale once per serialization. All changes are backward-compatible. | +| [v2.0.0](v2.0.0.md) | 2016-06-24 | Feature release with a minor (potentially non-backward-compatible) API change from added `noexcept` and `constexpr` specifiers. | + +## Version 1.1.x + +| Release | Date | Summary | +|---------|------|---------| +| [v1.1.0](v1.1.0.md) | 2016-01-24 | Bug-fix and feature release. All changes are backward-compatible. | + +## Version 1.0.x + +| Release | Date | Summary | +|---------|------|---------| +| [v1.0.0](v1.0.0.md) | 2015-12-28 | First official release. | + +## See also + +- [Migration Guide](../../integration/migration_guide.md) — how to future-proof your code for the next + major version and replace deprecated functions. diff --git a/docs/mkdocs/docs/home/releases/unreleased.md b/docs/mkdocs/docs/home/releases/unreleased.md new file mode 100644 index 000000000..b27da9748 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/unreleased.md @@ -0,0 +1,14 @@ +# Unreleased + +These changes are part of the `develop` branch, but not of a release yet. + +- Converting an rvalue container to JSON (`json j = std::move(vec);`, `json j = std::map{...};`) + now **moves** the elements instead of copying them (e.g., `std::vector` with 100,000 strings: 4.7 ms to + 2.3 ms). Two consequences: the source container is left in a valid but unspecified (moved-from) state, and if an + exception is thrown during the conversion, the source may be partially moved-from, so the strong exception guarantee + no longer holds for the source of an rvalue conversion (the JSON value itself is unaffected). Lvalues and `const` + objects are still copied. See [`basic_json`](../../api/basic_json/basic_json.md). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v1.0.0.md b/docs/mkdocs/docs/home/releases/v1.0.0.md new file mode 100644 index 000000000..dde38ca57 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v1.0.0.md @@ -0,0 +1,9 @@ +# v1.0.0 + +Released on 2015-12-28. + +First official release. [Full release notes](https://github.com/nlohmann/json/releases/tag/v1.0.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v1.1.0.md b/docs/mkdocs/docs/home/releases/v1.1.0.md new file mode 100644 index 000000000..1bdff4b59 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v1.1.0.md @@ -0,0 +1,14 @@ +# v1.1.0 + +Released on 2016-01-24. + +Bug-fix and feature release. All changes are backward-compatible. + +- Improves floating-point round-tripping, adds a `get_ref` accessor for stored values, and introduces + runtime [assertions](../../features/assertions.md). + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v1.1.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.0.md b/docs/mkdocs/docs/home/releases/v2.0.0.md new file mode 100644 index 000000000..540593592 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.0.md @@ -0,0 +1,19 @@ +# v2.0.0 + +Released on 2016-06-24. + +Feature release with a minor (potentially non-backward-compatible) API change from added `noexcept` +and `constexpr` specifiers. + +- Adds [JSON Pointer](../../features/json_pointer.md) support in [`at`](../../api/basic_json/at.md) and + [`operator[]`](../../api/basic_json/operator%5B%5D.md), plus [`flatten`](../../api/basic_json/flatten.md) + and [`unflatten`](../../api/basic_json/unflatten.md). +- Adds [JSON Patch](../../features/json_patch.md) via [`diff`](../../api/basic_json/diff.md) and + [`patch`](../../api/basic_json/patch.md), unsigned 64-bit integer support, and locale-independent + serialization. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.1.md b/docs/mkdocs/docs/home/releases/v2.0.1.md new file mode 100644 index 000000000..8d2820f14 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.1.md @@ -0,0 +1,11 @@ +# v2.0.1 + +Released on 2016-06-28. + +Fixes a performance regression in the [`dump`](../../api/basic_json/dump.md) function by adjusting the +stream locale once per serialization. All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.1). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.10.md b/docs/mkdocs/docs/home/releases/v2.0.10.md new file mode 100644 index 000000000..56e390f44 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.10.md @@ -0,0 +1,10 @@ +# v2.0.10 + +Released on 2017-01-02. + +Fixes several security-relevant bugs in the CBOR and MessagePack parsers found by OSS-Fuzz. All +changes are backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.10). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.2.md b/docs/mkdocs/docs/home/releases/v2.0.2.md new file mode 100644 index 000000000..4824db4a9 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.2.md @@ -0,0 +1,11 @@ +# v2.0.2 + +Released on 2016-07-31. + +Overhauls the parser (now rejecting unescaped control characters), tightens the class invariants, and +cleans up the code. All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.2). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.3.md b/docs/mkdocs/docs/home/releases/v2.0.3.md new file mode 100644 index 000000000..66182cb1d --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.3.md @@ -0,0 +1,11 @@ +# v2.0.3 + +Released on 2016-08-31. + +Generalizes the parser to accept any contiguous sequence of one-byte elements and deprecates the +input-stream constructor in favor of the [`parse`](../../api/basic_json/parse.md) function. All changes +are backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.3). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.4.md b/docs/mkdocs/docs/home/releases/v2.0.4.md new file mode 100644 index 000000000..b7cebb942 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.4.md @@ -0,0 +1,10 @@ +# v2.0.4 + +Released on 2016-09-11. + +Fixes stream end-of-file detection in the parser. All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.4). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.5.md b/docs/mkdocs/docs/home/releases/v2.0.5.md new file mode 100644 index 000000000..2f383efc9 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.5.md @@ -0,0 +1,10 @@ +# v2.0.5 + +Released on 2016-09-14. + +Fixes a remaining stream end-of-file detection bug in the parser. All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.5). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.6.md b/docs/mkdocs/docs/home/releases/v2.0.6.md new file mode 100644 index 000000000..cfee027fb --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.6.md @@ -0,0 +1,11 @@ +# v2.0.6 + +Released on 2016-10-15. + +Fixes [`operator[]`](../../api/basic_json/operator%5B%5D.md) for [JSON pointers](../../features/json_pointer.md) +so that it creates missing values like the other overloads. All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.6). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.7.md b/docs/mkdocs/docs/home/releases/v2.0.7.md new file mode 100644 index 000000000..3d8069ae6 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.7.md @@ -0,0 +1,11 @@ +# v2.0.7 + +Released on 2016-11-02. + +Fixes several parser bugs found through the "Parsing JSON is a Minefield" study (short files, encoding +detection, surrogate pairs). All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.7). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.8.md b/docs/mkdocs/docs/home/releases/v2.0.8.md new file mode 100644 index 000000000..177e335a5 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.8.md @@ -0,0 +1,12 @@ +# v2.0.8 + +Released on 2016-12-02. + +Adds the [`emplace`](../../api/basic_json/emplace.md) and +[`emplace_back`](../../api/basic_json/emplace_back.md) functions and improves parsing and serialization +performance. All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.8). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.0.9.md b/docs/mkdocs/docs/home/releases/v2.0.9.md new file mode 100644 index 000000000..2587c790e --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.0.9.md @@ -0,0 +1,11 @@ +# v2.0.9 + +Released on 2016-12-16. + +Adds the [CBOR](../../features/binary_formats/cbor.md) and +[MessagePack](../../features/binary_formats/messagepack.md) binary formats. All changes are +backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v2.0.9). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.1.0.md b/docs/mkdocs/docs/home/releases/v2.1.0.md new file mode 100644 index 000000000..3e42cbd32 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.1.0.md @@ -0,0 +1,15 @@ +# v2.1.0 + +Released on 2017-01-28. + +Feature release. All changes are backward-compatible. + +- Adds conversions from and to [arbitrary user-defined types](../../features/arbitrary_types.md) via + `to_json`/`from_json`, the [`meta`](../../api/basic_json/meta.md) function, and the option to switch off + exceptions ([`JSON_NOEXCEPTION`](../../api/macros/json_noexception.md)). + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.1.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v2.1.1.md b/docs/mkdocs/docs/home/releases/v2.1.1.md new file mode 100644 index 000000000..7167dcd16 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v2.1.1.md @@ -0,0 +1,14 @@ +# v2.1.1 + +Released on 2017-02-25. + +Bug-fix release. All changes are backward-compatible. + +- Makes number parsing and serialization locale-independent with correct floating-point + round-tripping; commits and released files are GPG-signed. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v2.1.1). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.0.0.md b/docs/mkdocs/docs/home/releases/v3.0.0.md new file mode 100644 index 000000000..cea928ee1 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.0.0.md @@ -0,0 +1,20 @@ +# v3.0.0 + +Released on 2017-12-17. + +First 3.x release — a major release with breaking changes (see the +[migration guide](../../integration/migration_guide.md)). + +- Introduces user-defined [exceptions](../exceptions.md) (`json::exception` and subtypes, each with an + identifier). +- Adds a non-throwing [`accept`](../../api/basic_json/accept.md) function and an `allow_exceptions` flag + for [`parse`](../../api/basic_json/parse.md), and an [`update`](../../api/basic_json/update.md) function to + merge objects. +- Adds streaming for CBOR and MessagePack and allows storing NaN/infinity. +- Non-UTF-8 strings now throw on serialization, and the iterator category changed to bidirectional. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.0.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.0.1.md b/docs/mkdocs/docs/home/releases/v3.0.1.md new file mode 100644 index 000000000..95c6141cf --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.0.1.md @@ -0,0 +1,12 @@ +# v3.0.1 + +Released on 2017-12-29. + +Fixes small issues in the [JSON Pointer](../../features/json_pointer.md) and +[JSON Patch](../../features/json_patch.md) implementations (invalid "copy" targets and non-integer array +indices). All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.0.1). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.1.0.md b/docs/mkdocs/docs/home/releases/v3.1.0.md new file mode 100644 index 000000000..31428a66f --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.1.0.md @@ -0,0 +1,16 @@ +# v3.1.0 + +Released on 2018-02-01. + +Feature release. All changes are backward-compatible. + +- Adds [UBJSON](../../features/binary_formats/ubjson.md) read/write support and + [JSON Merge Patch](../../features/merge_patch.md) via [`merge_patch`](../../api/basic_json/merge_patch.md). +- Switches to the Grisu2 algorithm for short, round-trippable floating-point output, and splits the + header into [multiple files](../../integration/index.md) with a forward-declaration header. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.1.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.1.1.md b/docs/mkdocs/docs/home/releases/v3.1.1.md new file mode 100644 index 000000000..b7c6aca1f --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.1.1.md @@ -0,0 +1,14 @@ +# v3.1.1 + +Released on 2018-02-13. + +Bug-fix release. All changes are backward-compatible. + +- Fixes parsing of indefinite-length CBOR strings, a user-defined conversion to vector types, and + overflow detection for UBJSON containers. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.1.1). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.1.2.md b/docs/mkdocs/docs/home/releases/v3.1.2.md new file mode 100644 index 000000000..30917e031 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.1.2.md @@ -0,0 +1,14 @@ +# v3.1.2 + +Released on 2018-03-14. + +Bug-fix release. All changes are backward-compatible. + +- Fixes a memory leak in the parser callback and adds user-defined string-type support to the parser + and serializer. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.1.2). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.10.0.md b/docs/mkdocs/docs/home/releases/v3.10.0.md new file mode 100644 index 000000000..520e8ab39 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.10.0.md @@ -0,0 +1,20 @@ +# v3.10.0 + +Released on 2021-08-17. + +Feature release. All changes are backward-compatible. + +- Adds [extended diagnostic messages](../../api/macros/json_diagnostics.md) + ([`JSON_DIAGNOSTICS`](../../api/macros/json_diagnostics.md)) that prepend a JSON pointer to exception + messages to pinpoint the offending value. +- Adds a GDB pretty-printer and a [`cbor_tag_handler_t`](../../api/basic_json/cbor_tag_handler_t.md) + `store` option to keep CBOR tags as binary subtypes. +- Supports containers with non-default-constructible types and parsing from `std::byte`. +- Adds [`JSON_NO_IO`](../../api/macros/json_no_io.md) to exclude the I/O headers and the + [`JSON_HAS_CPP_*`](../../api/macros/json_has_cpp_11.md) macros to override the detected C++ standard. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.10.1.md b/docs/mkdocs/docs/home/releases/v3.10.1.md new file mode 100644 index 000000000..e6881beee --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.10.1.md @@ -0,0 +1,14 @@ +# v3.10.1 + +Released on 2021-08-24. + +Fixes a regression from 3.10.0. All changes are backward-compatible. + +- Fixes an [extended-diagnostics](../../api/macros/json_diagnostics.md) assertion triggered when used + with [`ordered_json`](../../api/ordered_json.md), and hardens the GDB pretty-printer. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.1). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.10.2.md b/docs/mkdocs/docs/home/releases/v3.10.2.md new file mode 100644 index 000000000..c442ac668 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.10.2.md @@ -0,0 +1,11 @@ +# v3.10.2 + +Released on 2021-08-26. + +Re-release of 3.10.1, whose Git tag pointed at the wrong commit due to a bug in the release script. +All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.2). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.10.3.md b/docs/mkdocs/docs/home/releases/v3.10.3.md new file mode 100644 index 000000000..0c84ccfee --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.10.3.md @@ -0,0 +1,16 @@ +# v3.10.3 + +Released on 2021-10-08. + +Fixes more regressions from 3.10.0. All changes are backward-compatible. + +- Fixes [extended-diagnostics](../../api/macros/json_diagnostics.md) assertions triggered by + [`update`](../../api/basic_json/update.md) and by inserting into arrays. +- Supports custom allocators when writing binary formats into a `std::vector`, and allows conversion + from types that only provide `begin()`/`end()`. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.3). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.10.4.md b/docs/mkdocs/docs/home/releases/v3.10.4.md new file mode 100644 index 000000000..dcb81e38a --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.10.4.md @@ -0,0 +1,16 @@ +# v3.10.4 + +Released on 2021-10-16. + +Fixes regressions introduced in 3.10.0. All changes are backward-compatible. + +- Fixes the `std::filesystem::path` conversion (which could trigger a stack overflow and broke + compilation on Windows). +- Fixes compilation for types with an explicit defaulted constructor and for code relying on the + return values of `std::find` and `std::remove`. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.4). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.10.5.md b/docs/mkdocs/docs/home/releases/v3.10.5.md new file mode 100644 index 000000000..e1c82ba41 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.10.5.md @@ -0,0 +1,15 @@ +# v3.10.5 + +Released on 2022-01-03. + +Bug-fix release. All changes are backward-compatible. + +- Guards the `std::filesystem` conversions behind compiler-support checks + ([`JSON_HAS_FILESYSTEM`](../../api/macros/json_has_filesystem.md)), which can be set to `0` to disable + them altogether. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.10.5). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.11.0.md b/docs/mkdocs/docs/home/releases/v3.11.0.md new file mode 100644 index 000000000..c25a069a2 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.11.0.md @@ -0,0 +1,27 @@ +# v3.11.0 + +Released on 2022-08-01. + +One of the largest releases ever. All changes are backward-compatible. + +- Allows `std::string_view` as object keys in [`at`](../../api/basic_json/at.md), + [`operator[]`](../../api/basic_json/operator%5B%5D.md), [`value`](../../api/basic_json/value.md), + [`erase`](../../api/basic_json/erase.md), [`find`](../../api/basic_json/find.md), + [`contains`](../../api/basic_json/contains.md), and [`count`](../../api/basic_json/count.md). +- Adds the [BJData](../../features/binary_formats/bjdata.md) binary format (the fifth supported format). +- Improves C++20 support, including [`operator<=>`](../../api/basic_json/operator_spaceship.md) and + ``-compatible iterators. +- Adds a versioned, ABI-tagged inline [namespace](../../features/namespace.md) + ([`NLOHMANN_JSON_NAMESPACE`](../../api/macros/nlohmann_json_namespace.md)) and the option to move the + UDLs out of the global namespace ([`JSON_USE_GLOBAL_UDLS`](../../api/macros/json_use_global_udls.md)). +- Adds [`patch_inplace`](../../api/basic_json/patch_inplace.md), default values for the + [conversion macros](../../features/arbitrary_types.md#simplify-your-life-with-macros), and an option to + disable enum serialization ([`JSON_DISABLE_ENUM_SERIALIZATION`](../../api/macros/json_disable_enum_serialization.md)). + +This release introduced a UDL regression that was fixed in +[3.11.1](https://github.com/nlohmann/json/releases/tag/v3.11.1). +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.11.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.11.1.md b/docs/mkdocs/docs/home/releases/v3.11.1.md new file mode 100644 index 000000000..666a8951b --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.11.1.md @@ -0,0 +1,16 @@ +# v3.11.1 + +Released on 2022-08-01. + +Fixes a regression from 3.11.0. All changes are backward-compatible. + +- Restores the global [user-defined string literals](../../api/macros/json_use_global_udls.md) + [`operator""_json`](../../api/operator_literal_json.md) and + [`operator""_json_pointer`](../../api/operator_literal_json_pointer.md), which 3.11.0 had moved into a + namespace by default. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.11.1). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.11.2.md b/docs/mkdocs/docs/home/releases/v3.11.2.md new file mode 100644 index 000000000..a5f9e7ce3 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.11.2.md @@ -0,0 +1,18 @@ +# v3.11.2 + +Released on 2022-08-12. + +Fixes bugs found in 3.11.1 and restructures the namespace. All changes are backward-compatible. + +- Fixes the [`value`](../../api/basic_json/value.md) function (broken for strings, size types, and + `nullptr` in 3.11.0) and makes `json_fwd.hpp` self-contained. +- Restores using [`json_pointer`](../../api/json_pointer/index.md) as a key in associative containers and + comparing it with strings. +- Restructures the inline [namespace](../../features/namespace.md) and allows disabling the version + component, and avoids heap allocations in the [BJData](../../features/binary_formats/bjdata.md) parser. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.11.2). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.11.3.md b/docs/mkdocs/docs/home/releases/v3.11.3.md new file mode 100644 index 000000000..238a774c7 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.11.3.md @@ -0,0 +1,20 @@ +# v3.11.3 + +Released on 2023-11-28. + +Adds features and fixes bugs found in 3.11.2. All changes are backward-compatible. + +- Adds a [custom base class](../../api/basic_json/json_base_class_t.md) as a node customization point. +- Adds serialization-only [conversion macros](../../features/macros.md) + (`NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE` and `NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE`) + and a clearer parse error for empty input. +- Adds [Bazel](../../integration/package_managers.md#bazel) and + [Swift Package Manager](../../integration/package_managers.md#swift-package-manager) build support. +- Fixes custom allocators, a memory leak in [`adl_serializer`](../../api/adl_serializer/to_json.md)'s + `to_json`, initializer-list construction when `size_type` is not `int`, and many compiler warnings. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.11.3). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.12.0.md b/docs/mkdocs/docs/home/releases/v3.12.0.md new file mode 100644 index 000000000..07be55494 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.12.0.md @@ -0,0 +1,26 @@ +# v3.12.0 + +Released on 2025-04-11. + +Fixes bugs found in 3.11.3 and adds several features. No public API was removed or renamed. + +- Adds diagnostic byte positions via [`JSON_DIAGNOSTIC_POSITIONS`](../../api/macros/json_diagnostic_positions.md), + exposed through the new [`start_pos`](../../api/basic_json/start_pos.md) and + [`end_pos`](../../api/basic_json/end_pos.md) member functions. +- Makes the [conversion macros](../../features/arbitrary_types.md#simplify-your-life-with-macros) + templated (so they also work with [`ordered_json`](../../api/ordered_json.md)) and adds + [`NLOHMANN_DEFINE_DERIVED_TYPE`](../../api/macros/nlohmann_define_derived_type.md) for derived classes. +- Adds `std::optional` support (C++17) and lets [`patch`](../../api/basic_json/patch.md), + [`diff`](../../api/basic_json/diff.md), and [`flatten`](../../api/basic_json/flatten.md) work with + arbitrary string types. +- Extends the [binary formats](../../features/binary_formats/index.md): + [BJData](../../features/binary_formats/bjdata.md) draft 3 and unsigned 64-bit integers for + [BSON](../../features/binary_formats/bson.md). +- Adds multidimensional C-array conversion and UTF-8 encoded `std::filesystem::path` conversions, and + raises the minimum [CMake](../../integration/cmake.md) version to 3.5 and supports CMake 4.0. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.12.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.2.0.md b/docs/mkdocs/docs/home/releases/v3.2.0.md new file mode 100644 index 000000000..6e6690254 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.2.0.md @@ -0,0 +1,16 @@ +# v3.2.0 + +Released on 2018-08-20. + +Feature release. All changes are backward-compatible. + +- Adds a [SAX interface](../../features/parsing/sax_interface.md) and a non-recursive parser. +- Adds parsing from wide-string types (`std::wstring`, `std::u16string`, `std::u32string`), conversion to + `std::string_view` (C++17), and round-tripping of `std::map`/`std::unordered_map` with non-string + keys. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.2.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.3.0.md b/docs/mkdocs/docs/home/releases/v3.3.0.md new file mode 100644 index 000000000..e4ee07c84 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.3.0.md @@ -0,0 +1,14 @@ +# v3.3.0 + +Released on 2018-10-05. + +Feature release. All changes are backward-compatible. + +- Adds GCC 4.8 support, the [`get_to`](../../api/basic_json/get_to.md) function, and an overhauled and + documented [CMake](../../integration/cmake.md) integration. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.3.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.4.0.md b/docs/mkdocs/docs/home/releases/v3.4.0.md new file mode 100644 index 000000000..49a4cf595 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.4.0.md @@ -0,0 +1,18 @@ +# v3.4.0 + +Released on 2018-10-30. + +Feature release. All changes are backward-compatible. + +- Adds [BSON](../../features/binary_formats/bson.md) read/write support. +- Adds configurable Unicode error handlers to [`dump`](../../api/basic_json/dump.md) (throw, replace with + U+FFFD, or ignore) and the + [`NLOHMANN_JSON_SERIALIZE_ENUM`](../../api/macros/nlohmann_json_serialize_enum.md) macro for + [enum conversion](../../features/enum_conversion.md). +- Improves parse-error messages with line/column positions and context. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.4.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.5.0.md b/docs/mkdocs/docs/home/releases/v3.5.0.md new file mode 100644 index 000000000..40a14e159 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.5.0.md @@ -0,0 +1,15 @@ +# v3.5.0 + +Released on 2018-12-22. + +Feature release. All changes are backward-compatible. + +- Adds structured-binding support via the [`items`](../../api/basic_json/items.md) function and reading + from `FILE*` in the [`parse`](../../api/basic_json/parse.md) function. +- Fixes the `eofbit` handling on input streams and a bug in the BSON SAX parser. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.5.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.6.0.md b/docs/mkdocs/docs/home/releases/v3.6.0.md new file mode 100644 index 000000000..48aa6e9b9 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.6.0.md @@ -0,0 +1,16 @@ +# v3.6.0 + +Released on 2019-03-20. + +Feature release. All changes are backward-compatible. + +- Reworks the [JSON pointer](../../features/json_pointer.md) interface (`operator/`, `push_back`, + `parent_pointer`, …). +- Adds a [`contains`](../../api/basic_json/contains.md) function to test for an object key and greatly + improves the performance of integer serialization. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.6.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.6.1.md b/docs/mkdocs/docs/home/releases/v3.6.1.md new file mode 100644 index 000000000..c1e8daf47 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.6.1.md @@ -0,0 +1,10 @@ +# v3.6.1 + +Released on 2019-03-20. + +Fixes a regression (GCC 7/8 compilation) and a `` build error introduced in 3.6.0. All +changes are backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v3.6.1). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.7.0.md b/docs/mkdocs/docs/home/releases/v3.7.0.md new file mode 100644 index 000000000..c9e7b15b6 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.7.0.md @@ -0,0 +1,15 @@ +# v3.7.0 + +Released on 2019-07-28. + +Convenience features and house-keeping. All changes are backward-compatible. + +- Adds a [`contains`](../../api/basic_json/contains.md) overload that checks a JSON pointer without + throwing, a generic `to_string`, and a return value for + [`emplace_back`](../../api/basic_json/emplace_back.md). + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.7.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.7.1.md b/docs/mkdocs/docs/home/releases/v3.7.1.md new file mode 100644 index 000000000..98a85fb68 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.7.1.md @@ -0,0 +1,16 @@ +# v3.7.1 + +Released on 2019-11-06. + +Bug-fix release. All changes are backward-compatible. + +- Fixes a segmentation fault when serializing the `std::int64_t` minimum value and fixes + [`contains`](../../api/basic_json/contains.md) for JSON pointers. +- Allows [`items`](../../api/basic_json/items.md) with a custom string type and makes `json_pointer::back` + `const`. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.7.1). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.7.2.md b/docs/mkdocs/docs/home/releases/v3.7.2.md new file mode 100644 index 000000000..aea237be8 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.7.2.md @@ -0,0 +1,11 @@ +# v3.7.2 + +Released on 2019-11-10. + +Fixes a stack overflow for deeply nested input by making the destructor iterative; parsing of JSON +text is now bounded only by available memory (the binary-format parsers still recurse). All changes are backward-compatible. +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.7.2). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.7.3.md b/docs/mkdocs/docs/home/releases/v3.7.3.md new file mode 100644 index 000000000..f0a106f39 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.7.3.md @@ -0,0 +1,10 @@ +# v3.7.3 + +Released on 2019-11-17. + +Fixes a regression from 3.7.2 that could yield quadratic complexity in destructor calls. All changes +are backward-compatible. [Full release notes](https://github.com/nlohmann/json/releases/tag/v3.7.3). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.8.0.md b/docs/mkdocs/docs/home/releases/v3.8.0.md new file mode 100644 index 000000000..babd55fac --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.8.0.md @@ -0,0 +1,20 @@ +# v3.8.0 + +Released on 2020-06-14. + +Feature release. All changes are backward-compatible. + +- Introduces a [binary value](../../features/binary_values.md) type that is read from and written to + [CBOR](../../features/binary_formats/cbor.md), [BSON](../../features/binary_formats/bson.md), and + [MessagePack](../../features/binary_formats/messagepack.md), and can be shared between formats. +- Generalizes the input adapters to read from any `LegacyInputIterator` container (3–10 % faster + parsing). +- Fixes [`contains`](../../api/basic_json/contains.md) for JSON pointers and makes the binary + [`from_cbor`](../../api/basic_json/from_cbor.md)/[`from_msgpack`](../../api/basic_json/from_msgpack.md)/etc. + functions respect `allow_exceptions`. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.8.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.9.0.md b/docs/mkdocs/docs/home/releases/v3.9.0.md new file mode 100644 index 000000000..c04668c62 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.9.0.md @@ -0,0 +1,20 @@ +# v3.9.0 + +Released on 2020-07-27. + +Feature release adding four long-requested features. All changes are backward-compatible. + +- Optional [comment](../../features/comments.md) parsing in [`parse`](../../api/basic_json/parse.md) via the + `ignore_comments` parameter. +- [`ordered_json`](../../api/ordered_json.md) to preserve the [insertion order](../../features/object_order.md) + of object keys. +- An option to switch off [implicit conversions](../../api/macros/json_use_implicit_conversions.md). +- The [`NLOHMANN_DEFINE_TYPE_*`](../../features/arbitrary_types.md#simplify-your-life-with-macros) + convenience macros, plus high-precision-number support for + [UBJSON](../../features/binary_formats/ubjson.md) and CBOR tag handling. + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.9.0). + +## API changes + + diff --git a/docs/mkdocs/docs/home/releases/v3.9.1.md b/docs/mkdocs/docs/home/releases/v3.9.1.md new file mode 100644 index 000000000..a06f819d7 --- /dev/null +++ b/docs/mkdocs/docs/home/releases/v3.9.1.md @@ -0,0 +1,14 @@ +# v3.9.1 + +Released on 2020-08-06. + +Fixes two regressions from 3.9.0. All changes are backward-compatible. + +- Accepts consecutive [comments](../../features/comments.md) and completes the + [`ordered_json`](../../api/ordered_json.md) interface (e.g. `ordered_json::parse`). + +[Full release notes](https://github.com/nlohmann/json/releases/tag/v3.9.1). + +## API changes + + diff --git a/docs/mkdocs/docs/images/range-begin-end.svg b/docs/mkdocs/docs/images/range-begin-end.svg index 8e2b2fb6c..99c9f8662 100644 --- a/docs/mkdocs/docs/images/range-begin-end.svg +++ b/docs/mkdocs/docs/images/range-begin-end.svg @@ -16,6 +16,18 @@ version="1.1" inkscape:version="0.48.4 r9939" sodipodi:docname="range-begin-end.svg"> + + unreleased badge; see - [Releases](home/releases.md) for what shipped in each version. + [Releases](home/releases/index.md) for what shipped in each version. The library is licensed under the [MIT License](home/license.md). The source code, issue tracker, and discussions are on [GitHub](https://github.com/nlohmann/json). diff --git a/docs/mkdocs/docs/integration/index.md b/docs/mkdocs/docs/integration/index.md index f022ec6ed..f6f303142 100644 --- a/docs/mkdocs/docs/integration/index.md +++ b/docs/mkdocs/docs/integration/index.md @@ -54,3 +54,12 @@ For the read-only, non-owning [`basic_json_document`](../api/basic_json_document types, additionally include [`single_include/nlohmann/json_view.hpp`](https://github.com/nlohmann/json/blob/develop/single_include/nlohmann/json_view.hpp); see [Zero-copy JSON views](../features/json_view.md). + +## See also + +- [CMake](cmake.md) - integrate the library into a CMake project +- [Package managers](package_managers.md) +- [pkg-config](pkg-config.md) +- [Compile times](compile_times.md) - reduce the compile time overhead +- [Migration guide](migration_guide.md) - upgrade from an earlier version +- [Releases](../home/releases/index.md) diff --git a/docs/mkdocs/hooks/api_changes.py b/docs/mkdocs/hooks/api_changes.py new file mode 100644 index 000000000..94073f534 --- /dev/null +++ b/docs/mkdocs/hooks/api_changes.py @@ -0,0 +1,163 @@ +"""Generate the "API changes" section of the release pages from the version histories of the API pages.""" + +# Every page below docs/api ends in a "Version history" section ("Added in version 3.8.0.", "... since version 3.13.0."). +# This hook collects these entries and replaces the `` marker of each page in home/releases with +# the pages that were added in that release and the changes made to existing pages. The page of a release is named +# after its version (v3.12.0.md); unreleased.md collects all versions newer than the latest release. The result is an +# overview of what changed in the API, not a change log. + +import logging +import os +import posixpath +import re + +log = logging.getLogger("mkdocs.hooks.api_changes") + +_DIR = "home/releases" +_PAGE_VERSION = re.compile(r"v(\d+)\.(\d+)\.(\d+)\.md") +_MARKER = "" +_HISTORY = re.compile(r"^## Version history\n(.*?)(?=^## |\Z)", re.DOTALL | re.MULTILINE) +_TITLE = re.compile(r"^# (.+)$", re.MULTILINE) +_ITEM = re.compile(r"^(?:[-*]|\d+\.)\s+") +_SENTENCE = re.compile(r"(?<=\.)\s+(?=[A-Z`\[])") +_VERSION = re.compile(r"\b(?:version|since|in)\s+(\d+)\.(\d+)\.(\d+)\b", re.IGNORECASE) +_FUTURE = re.compile(r"\bwill\b|\bplanned\b", re.IGNORECASE) # statements about versions that do not exist yet +_NEXT_MAJOR = (4, 0, 0) # versions from here on are announcements, not changes +_ADDED = re.compile(r"^(?:Added|Introduced)\b.*\bversion \d|^Since version \d+\.\d+\.\d+\.?$", re.IGNORECASE) +_LINK = re.compile(r"(\]\()([^)\s]+)(\))") + + +def on_page_markdown(markdown, *, page, config, files): + uri = page.file.src_uri + if posixpath.dirname(uri) != _DIR or _MARKER not in markdown: + return markdown + docs_dir = config["docs_dir"] + entries = _collect(docs_dir) + released = sorted(tuple(map(int, m.groups())) for m in + (_PAGE_VERSION.fullmatch(name) for name in os.listdir(os.path.join(docs_dir, _DIR))) if m) + match = _PAGE_VERSION.fullmatch(posixpath.basename(uri)) + if match: + versions = [tuple(map(int, match.groups()))] + else: # unreleased.md + versions = sorted((v for v in entries if not released or v > released[-1]), reverse=True) + return markdown.replace(_MARKER, _render(uri, [(v, entries[v]) for v in versions if v in entries])) + + +def _title(text, path): + match = _TITLE.search(text) + title = match.group(1) if match else os.path.splitext(os.path.basename(path))[0] + title = re.sub(r"", "", title) + title = title.replace("`", "").replace("\\", "").strip() + return title[len("nlohmann::"):] if title.startswith("nlohmann::") else title + + +def _collect(docs_dir): + """Return {version: {"added": [(title, uri)], "changed": [(title, uri, text)]}}.""" + result = {} + api = os.path.join(docs_dir, "api") + for root, _, names in sorted(os.walk(api)): + for name in sorted(names): + if not name.endswith(".md"): + continue + path = os.path.join(root, name) + uri = posixpath.relpath(path.replace(os.sep, "/"), docs_dir.replace(os.sep, "/")) + with open(path, encoding="utf-8") as source: + text = source.read() + history = _HISTORY.search(text) + if not history: + continue + title = _title(text, path) + found = _sentences(history.group(1)) + # a page is added in the earliest version named by an "Added in" sentence that opens an entry (an entry of + # a numbered list describes one overload; later unnumbered entries extend an existing page) + starts = [v for i, n, first, v, sentence in found if first and _ADDED.match(sentence) and (i == 0 or n)] + added = min(starts) if starts else None + page_added = False + for index, number, first, version, sentence in found: + if version >= _NEXT_MAJOR: + continue + bucket = result.setdefault(version, {"added": [], "changed": []}) + opens = first and _ADDED.match(sentence) and (index == 0 or number) + if opens and version == added: + if not page_added: + bucket["added"].append((title, uri)) + page_added = True + continue + if number and re.fullmatch(r"Added in version [\d.]+\.?", sentence): + sentence = f"Overload {number} added." + elif number: + sentence = f"Overload {number}: {sentence}" + if (title, uri, sentence) not in bucket["changed"]: + bucket["changed"].append((title, uri, sentence)) + return result + + +def _sentences(section): + """Yield (entry index, overload number, opens entry, version, sentence) for every sentence naming a version.""" + found = [] + for index, (number, entry) in enumerate(_entries(section)): + for position, sentence in enumerate(_SENTENCE.split(entry)): + version = _VERSION.search(sentence) + if not version or _FUTURE.search(sentence): + continue + found.append((index, number, position == 0, tuple(int(part) for part in version.groups()), sentence.strip())) + return found + + +def _entries(section): + entries = [] + for line in section.split("\n"): + item = _ITEM.match(line) + if item: + number = re.match(r"\d+", line) + entries.append([number.group(0) if number else None, _ITEM.sub("", line).strip()]) + elif line.strip() and entries and line[0].isspace(): + entries[-1][1] += " " + line.strip() # wrapped list item + elif line.strip(): + break # text after the list, such as an admonition + return [tuple(entry) for entry in entries] + + +def _relink(text, uri, page): + """Rewrite links relative to the API page `uri` so they work from the generated page.""" + def fix(match): + target = match.group(2) + if re.match(r"[a-z]+:|#|/", target): + return match.group(0) + resolved = posixpath.normpath(posixpath.join(posixpath.dirname(uri), target)) + return match.group(1) + posixpath.relpath(resolved, posixpath.dirname(page)) + match.group(3) + return _LINK.sub(fix, text) + + +def _link(title, uri, page): + return f"[`{title}`]({posixpath.relpath(uri, posixpath.dirname(page))})" + + +def _added(pages, page): + """Group the added pages by class: `basic_json`: [`get`](..), ...; pages without a class come first.""" + groups, single = {}, [] + for title, uri in pages: + directory = posixpath.dirname(uri) + if directory == "api": + single.append(_link(title, uri, page)) + elif posixpath.basename(uri) == "index.md": + groups.setdefault(directory, []).insert(0, f"[(overview)]({posixpath.relpath(uri, posixpath.dirname(page))})") + else: + groups.setdefault(directory, []).append(_link(title.split("::")[-1], uri, page)) + lines = [f"- {item}" for item in single] + for directory, items in groups.items(): + lines.append(f"- `{posixpath.basename(directory)}`: " + ", ".join(items)) + return lines + + +def _render(page, entries): + lines = [] + for version, bucket in entries: + if len(entries) > 1: + lines += [f"### Version {'.'.join(map(str, version))}", ""] + if bucket["added"]: + lines += [f"**Added ({len(bucket['added'])} pages):**", ""] + _added(bucket["added"], page) + [""] + if bucket["changed"]: + lines += ["**Changed:**", ""] + lines += [f"- {_link(t, u, page)}: {_relink(x, u, page)}" for t, u, x in bucket["changed"]] + [""] + return "\n".join(lines) if lines else "No documented API changes." diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index 90393ff0d..469ae9260 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -51,7 +51,53 @@ nav: - home/license.md - "FAQ": home/faq.md - home/exceptions.md - - home/releases.md + - Releases: + - 'Overview': home/releases/index.md + - 'Unreleased': home/releases/unreleased.md + - 'v3.12.0': home/releases/v3.12.0.md + - 'v3.11.3': home/releases/v3.11.3.md + - 'v3.11.2': home/releases/v3.11.2.md + - 'v3.11.1': home/releases/v3.11.1.md + - 'v3.11.0': home/releases/v3.11.0.md + - 'v3.10.5': home/releases/v3.10.5.md + - 'v3.10.4': home/releases/v3.10.4.md + - 'v3.10.3': home/releases/v3.10.3.md + - 'v3.10.2': home/releases/v3.10.2.md + - 'v3.10.1': home/releases/v3.10.1.md + - 'v3.10.0': home/releases/v3.10.0.md + - 'v3.9.1': home/releases/v3.9.1.md + - 'v3.9.0': home/releases/v3.9.0.md + - 'v3.8.0': home/releases/v3.8.0.md + - 'v3.7.3': home/releases/v3.7.3.md + - 'v3.7.2': home/releases/v3.7.2.md + - 'v3.7.1': home/releases/v3.7.1.md + - 'v3.7.0': home/releases/v3.7.0.md + - 'v3.6.1': home/releases/v3.6.1.md + - 'v3.6.0': home/releases/v3.6.0.md + - 'v3.5.0': home/releases/v3.5.0.md + - 'v3.4.0': home/releases/v3.4.0.md + - 'v3.3.0': home/releases/v3.3.0.md + - 'v3.2.0': home/releases/v3.2.0.md + - 'v3.1.2': home/releases/v3.1.2.md + - 'v3.1.1': home/releases/v3.1.1.md + - 'v3.1.0': home/releases/v3.1.0.md + - 'v3.0.1': home/releases/v3.0.1.md + - 'v3.0.0': home/releases/v3.0.0.md + - 'v2.1.1': home/releases/v2.1.1.md + - 'v2.1.0': home/releases/v2.1.0.md + - 'v2.0.10': home/releases/v2.0.10.md + - 'v2.0.9': home/releases/v2.0.9.md + - 'v2.0.8': home/releases/v2.0.8.md + - 'v2.0.7': home/releases/v2.0.7.md + - 'v2.0.6': home/releases/v2.0.6.md + - 'v2.0.5': home/releases/v2.0.5.md + - 'v2.0.4': home/releases/v2.0.4.md + - 'v2.0.3': home/releases/v2.0.3.md + - 'v2.0.2': home/releases/v2.0.2.md + - 'v2.0.1': home/releases/v2.0.1.md + - 'v2.0.0': home/releases/v2.0.0.md + - 'v1.1.0': home/releases/v1.1.0.md + - 'v1.0.0': home/releases/v1.0.0.md - home/design_goals.md - home/architecture.md - home/customers.md @@ -484,6 +530,7 @@ validation: unrecognized_links: warn hooks: + - hooks/api_changes.py - hooks/copy_markdown_source.py - hooks/unreleased_versions.py diff --git a/docs/mkdocs/scripts/mermaid/package-lock.json b/docs/mkdocs/scripts/mermaid/package-lock.json index d7d3cad06..14bbc249e 100644 --- a/docs/mkdocs/scripts/mermaid/package-lock.json +++ b/docs/mkdocs/scripts/mermaid/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "check-mermaid", "dependencies": { - "jsdom": "30.1.2", + "jsdom": "30.1.3", "mermaid": "11.17.2" } }, @@ -40,9 +40,9 @@ } }, "node_modules/@asamuzakjp/dom-selector": { - "version": "9.2.4", - "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-9.2.4.tgz", - "integrity": "sha512-YnVzxDxLaqL0t7Q4wfcgHZjg55Wm6y5EaO7Bhu3q9J7wq/8iw3PhsnWiOQhYekx4VexnbkgWpTT6n+QIV7kczw==", + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-10.0.3.tgz", + "integrity": "sha512-3+777rEPsOOjoy5/sRmIzHnoRDaLpm3XWY2dmWn/N7f/XwDG0bcpnFaKL3X9FYJUbNwLmpcJC1bUYexJRxsFlw==", "license": "MIT", "dependencies": { "bidi-js": "^1.1.0", @@ -1105,9 +1105,9 @@ } }, "node_modules/dompurify": { - "version": "3.4.16", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.16.tgz", - "integrity": "sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ==", + "version": "3.4.17", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.17.tgz", + "integrity": "sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -1202,13 +1202,13 @@ "license": "MIT" }, "node_modules/jsdom": { - "version": "30.1.2", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.1.2.tgz", - "integrity": "sha512-0FFE/jE1rppmVfUrJUgxqXjcwolZYIGtAgj1pTussMhNZxIxi7W/PvfWaMNroGi2B36X1IKHbexpZ9DhkoOPiQ==", + "version": "30.1.3", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.1.3.tgz", + "integrity": "sha512-hQMUPvRgN7KxsfIytDQji7bimuJeRQa6Qo5+iuiZQk5CafeRxLIA10/cG5YCBFNpD9RO3oqIiN4CO1Y44kxqXg==", "license": "MIT", "dependencies": { "@asamuzakjp/css-color": "^7.1.3", - "@asamuzakjp/dom-selector": "^9.2.4", + "@asamuzakjp/dom-selector": "^10.0.3", "@bramus/specificity": "^2.4.2", "@csstools/css-syntax-patches-for-csstree": "^1.1.15", "@exodus/bytes": "^1.16.0", @@ -1221,7 +1221,7 @@ "parse5": "^8.0.1", "saxes": "^6.0.0", "tough-cookie": "^6.0.2", - "undici": "^8.11.2", + "undici": "^8.12.1", "w3c-xmlserializer": "^6.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", @@ -1521,9 +1521,9 @@ } }, "node_modules/undici": { - "version": "8.11.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.11.2.tgz", - "integrity": "sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ==", + "version": "8.12.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.12.2.tgz", + "integrity": "sha512-XtuDWVdqTLr3NAxm2ly9/hU6kUsJOgGh1RASzMBbsw6aWCOaQzHdW+QQDfdERjfJ2V5vL5pRPa4RetO9UfivDg==", "license": "MIT", "engines": { "node": ">=22.19.0" diff --git a/docs/mkdocs/scripts/mermaid/package.json b/docs/mkdocs/scripts/mermaid/package.json index c79f4fa89..d2c033d3d 100644 --- a/docs/mkdocs/scripts/mermaid/package.json +++ b/docs/mkdocs/scripts/mermaid/package.json @@ -4,7 +4,7 @@ "description": "Validate the Mermaid diagrams of the documentation (see check_mermaid.mjs)", "type": "module", "dependencies": { - "jsdom": "30.1.2", + "jsdom": "30.1.3", "mermaid": "11.17.2" } } diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index fef2c0f93..6c7168d47 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -86,7 +86,7 @@ JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 2 /*! @brief deserialization of BJData, BON8, BSON, CBOR, MessagePack, and UBJSON values */ -template> +template> class binary_reader { using number_integer_t = typename BasicJsonType::number_integer_t; diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 77a1b45c5..05fd4c816 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -8,17 +8,18 @@ #pragma once -#include // count, find, min +#include // count, find, max, min #include // array #include // size_t #include // uint32_t #include // strlen #include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator #include // numeric_limits +#include // unique_ptr #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer -#include // pair, declval +#include // declval, move, pair #include // vector #ifndef JSON_NO_IO @@ -72,6 +73,82 @@ enum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata, bon8 }; #ifndef JSON_NO_IO /*! +A growable array of characters for the part of a token that an input adapter +copied out of its window: unlike std::vector::push_back(), appending a +character takes no reference to a local, and is a few instructions when there +is room. +*/ +class token_carry_buffer +{ + public: + token_carry_buffer() = default; + token_carry_buffer(const token_carry_buffer&) = delete; + token_carry_buffer& operator=(const token_carry_buffer&) = delete; + token_carry_buffer(token_carry_buffer&& rhs) noexcept + : m_data(std::move(rhs.m_data)), m_end(rhs.m_end), m_capacity_end(rhs.m_capacity_end) + { + rhs.m_end = rhs.m_capacity_end = nullptr; + } + token_carry_buffer& operator=(token_carry_buffer&&) = delete; + ~token_carry_buffer() = default; + + void clear() noexcept + { + m_end = m_data.get(); + } + + void push_back(char c) + { + if (JSON_HEDLEY_UNLIKELY(m_end == m_capacity_end)) + { + grow(1); + } + *m_end++ = c; + } + + void append(const char* first, const char* last) + { + const auto n = static_cast(last - first); + const auto room = static_cast(m_capacity_end - m_end); + if (n > room) + { + grow(n); + } + if (n != 0) + { + std::memcpy(m_end, first, n); + } + m_end += n; + } + + /// replace the contents of @a out + void copy_to(std::vector& out) const + { + out.assign(m_data.get(), m_end); + } + + private: + /// make room for @a extra more characters + JSON_HEDLEY_NEVER_INLINE void grow(std::size_t extra) + { + const auto size = static_cast(m_end - m_data.get()); + const auto capacity = static_cast(m_capacity_end - m_data.get()); + const std::size_t new_capacity = (std::max)(capacity < 32 ? static_cast(32) : 2 * capacity, size + extra); + std::unique_ptr data(new char[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + if (size != 0) + { + std::memcpy(data.get(), m_data.get(), size); + } + m_data = std::move(data); + m_end = m_data.get() + size; + m_capacity_end = m_data.get() + new_capacity; + } + + std::unique_ptr m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init) + char* m_end = nullptr; + char* m_capacity_end = nullptr; +}; +/*! Input adapter for stdio file access. This adapter read only 1 byte and do not use any buffer. This adapter is a very low level adapter. */ @@ -209,6 +286,7 @@ class input_stream_adapter , token_segment(rhs.token_segment) , current_in_get_area(rhs.current_in_get_area) , tracking_token(rhs.tracking_token) + , no_get_area(rhs.no_get_area) #endif { rhs.is = nullptr; @@ -295,37 +373,20 @@ class input_stream_adapter std::char_traits::int_type get_character() { #if !JSON_DISABLE_STREAM_BULK_SCAN - // sbumpc() refills the get area if it is exhausted - const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); - if (JSON_HEDLEY_UNLIKELY(refill)) - { - save_token_segment(); - } -#endif - - auto res = sb->sbumpc(); - // set eof manually, as we don't use the istream interface. - if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) - { -#if !JSON_DISABLE_STREAM_BULK_SCAN - current_in_get_area = false; -#endif - is->clear(is->rdstate() | std::ios::eofbit); - } -#if !JSON_DISABLE_STREAM_BULK_SCAN - else if (JSON_HEDLEY_UNLIKELY(refill)) - { - // the consumed character is at gptr() - 1, unless the streambuf - // has no get area and returned it from uflow() - const char* const next = streambuf_access::get_ptr(sb); - after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); - } - else + // the character is in the get area: step over it, as sbumpc() does + const char* const next = streambuf_access::get_ptr(sb); + if (JSON_HEDLEY_LIKELY(next != streambuf_access::get_end(sb))) { current_in_get_area = true; + streambuf_access::bump(sb, 1); + return std::char_traits::to_int_type(*next); + } + if (no_get_area) + { + return get_character_no_get_area(); } #endif - return res; + return get_character_slow(); } #endif @@ -334,6 +395,18 @@ class input_stream_adapter { #if JSON_PRECISE_STREAM_POSITION commit_lookahead(); +#endif +#if !JSON_DISABLE_STREAM_BULK_SCAN + // the elements are in the get area: copy them, as sgetn() does, but + // without the virtual call + const char* const first = streambuf_access::get_ptr(sb); + const auto available = static_cast(streambuf_access::get_end(sb) - first); + if (JSON_HEDLEY_LIKELY(count * sizeof(T) <= available && count * sizeof(T) <= static_cast((std::numeric_limits::max)()))) + { + std::memcpy(dest, first, count * sizeof(T)); + streambuf_access::bump(sb, static_cast(count * sizeof(T))); + return count * sizeof(T); + } #endif auto res = static_cast(sb->sgetn(reinterpret_cast(dest), static_cast(count * sizeof(T)))); if (JSON_HEDLEY_UNLIKELY(res < count * sizeof(T))) @@ -421,7 +494,7 @@ class input_stream_adapter // last one if @a drop_last. void copy_token(std::vector& out, bool drop_last) const { - out = token_carry; + token_carry.copy_to(out); if (token_segment != nullptr) { out.insert(out.end(), token_segment, current_end()); @@ -434,6 +507,47 @@ class input_stream_adapter #endif private: +#if !JSON_PRECISE_STREAM_POSITION + // the get area is exhausted, or there is none +#if !JSON_DISABLE_STREAM_BULK_SCAN + JSON_HEDLEY_NEVER_INLINE +#endif + std::char_traits::int_type get_character_slow() + { +#if !JSON_DISABLE_STREAM_BULK_SCAN + // sbumpc() refills the get area if it is exhausted + const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); + if (JSON_HEDLEY_UNLIKELY(refill)) + { + save_token_segment(); + } +#endif + + auto res = sb->sbumpc(); + // set eof manually, as we don't use the istream interface. + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) + { +#if !JSON_DISABLE_STREAM_BULK_SCAN + current_in_get_area = false; +#endif + is->clear(is->rdstate() | std::ios::eofbit); + } +#if !JSON_DISABLE_STREAM_BULK_SCAN + else if (JSON_HEDLEY_UNLIKELY(refill)) + { + // the consumed character is at gptr() - 1, unless the streambuf + // has no get area and returned it from uflow() + const char* const next = streambuf_access::get_ptr(sb); + after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); + } + else + { + current_in_get_area = true; + } +#endif + return res; + } +#endif #if !JSON_DISABLE_STREAM_BULK_SCAN // Reads the protected get-area pointers of any std::streambuf. Naming the // members through a derived class makes forming the member pointers @@ -457,6 +571,11 @@ class input_stream_adapter { (s->*(&streambuf_access::gbump))(n); } + // sbumpc() for an exhausted get area + static std::streambuf::int_type next_character(std::streambuf* s) + { + return (s->*(&streambuf_access::uflow))(); + } }; // one past the character last returned by get_character(), if that is in @@ -477,16 +596,53 @@ class input_stream_adapter { if (token_segment != nullptr) { - token_carry.insert(token_carry.end(), token_segment, streambuf_access::get_end(sb)); + token_carry.append(token_segment, streambuf_access::get_end(sb)); token_segment = nullptr; } } +#if !JSON_PRECISE_STREAM_POSITION + // The streambuf has no get area (e.g., std::cin synchronized with stdio): + // every character comes from uflow(), and the token is kept by value. + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_no_get_area() + { + // the get area is exhausted, so this is what sbumpc() does + const auto res = streambuf_access::next_character(sb); + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof() || streambuf_access::get_end(sb) != nullptr)) + { + return get_character_no_get_area_other(res); + } + // current_in_get_area is false: only the first character of a get + // area, or the end of the input, is read here + if (tracking_token) + { + token_carry.push_back(std::char_traits::to_char_type(res)); + } + return res; + } + + // @a res came from uflow() of a streambuf without a get area: it is the + // end of the input, or the streambuf has a get area now + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_no_get_area_other(std::char_traits::int_type res) + { + if (res == std::char_traits::eof()) + { + current_in_get_area = false; + is->clear(is->rdstate() | std::ios::eofbit); + return res; + } + // the consumed character is at gptr() - 1 + after_refill(res, streambuf_access::get_ptr(sb) != streambuf_access::get_begin(sb)); + return res; + } +#endif + // character @a c was just read into a new get area (@a in_get_area) or // returned without one void after_refill(std::char_traits::int_type c, bool in_get_area) { current_in_get_area = in_get_area; + no_get_area = !in_get_area; if (!tracking_token) { return; @@ -525,13 +681,16 @@ class input_stream_adapter #endif #if !JSON_DISABLE_STREAM_BULK_SCAN /// the part of the current token that was in earlier get areas - std::vector token_carry{}; // NOLINT(readability-redundant-member-init) + token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init) /// start of the current token's part in the get area, or nullptr const char* token_segment = nullptr; /// whether the character last returned by get_character() is in the get area bool current_in_get_area = false; /// whether a token is tracked at all; the binary readers never start one bool tracking_token = false; + /// whether the streambuf had no get area when it was last asked for a + /// character + bool no_get_area = false; #endif }; #endif // JSON_NO_IO diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 966610e4a..db95685f0 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -8,9 +8,13 @@ #pragma once -#include // find_if, min +#include // copy, find_if, min +#include // array #include +#include // hash #include // numeric_limits +#include // unique_ptr +#include // operator new #include // string #include // enable_if_t #include // move, pair @@ -18,8 +22,10 @@ #include #include +#include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -315,20 +321,114 @@ struct diagnostic_positions }; #endif +/*! +@brief a hash table that finds the position of a key in an object whose + elements are kept in a vector (ordered_map) + +The elements stay where they are: the table only holds their hashes and +positions, and the caller compares the keys. +*/ +class object_key_index +{ + public: + static constexpr std::size_t npos = static_cast(-1); + + /// @param[in] element_hashes the hashes of the keys of the elements that are there already + explicit object_key_index(std::vector&& element_hashes) + : hashes(std::move(element_hashes)) + { + std::size_t size = 256; + while (size < 4 * hashes.size()) + { + size *= 2; + } + rebuild(size); + } + + /// @return the position of the element with @a hash for which @a matches(position) holds, or npos + template + std::size_t find(const std::size_t hash, Matches matches) const + { + for (std::size_t slot = hash & mask;; slot = (slot + 1) & mask) + { + const std::size_t entry = table[slot]; + if (entry == 0) + { + return npos; + } + if (hashes[entry - 1] == hash && matches(entry - 1)) + { + return entry - 1; + } + } + } + + /// add the next element, whose position is the number of elements so far + void add(const std::size_t hash) + { + hashes.push_back(hash); + if (hashes.size() * 2 > table.size()) + { + rebuild(table.size() * 2); + } + else + { + place(hash, hashes.size()); + } + } + + private: + void rebuild(const std::size_t size) + { + table.assign(size, 0); + mask = size - 1; + for (std::size_t i = 0; i < hashes.size(); ++i) + { + place(hashes[i], i + 1); + } + } + + void place(const std::size_t hash, const std::size_t entry) + { + std::size_t slot = hash & mask; + while (table[slot] != 0) + { + slot = (slot + 1) & mask; + } + table[slot] = entry; + } + + /// the hash of each element, by position + std::vector hashes; + /// open addressing, a power of two in size: position + 1 of an element, or 0 for a free slot + std::vector table {}; // NOLINT(readability-redundant-member-init) + std::size_t mask = 0; +}; + /*! @brief SAX implementation to create a JSON value from SAX events This class implements the @ref json_sax interface and processes the SAX events to create a JSON value which makes it basically a DOM parser. The structure or -hierarchy of the JSON value is managed by the stack `ref_stack` which contains -a pointer to the respective array or object for each recursion depth. +hierarchy of the JSON value is managed by the stack `frames` which contains +an entry for each open array or object. The parser of the text formats, for +which the size of a container is not known up front, collects the elements of +an array and the members of an `ordered_json` object on a stack and moves them +into a container of the exact size (with @a Collect, as basic_json's own parse +functions use it); the binary readers use the plain +@ref json_sax_dom_binary_parser instead. After successful parsing, the value that is passed by reference to the -constructor contains the parsed value. +constructor contains the parsed value. Without @a Collect, every value is +stored in place as it arrives, so after a parse error the value holds what was +read up to it. @tparam BasicJsonType the JSON type +@tparam Collect whether to collect the values of containers of unknown size on + a stack (only for a caller that does not use the value after a + parse error) */ -template +template class json_sax_dom_parser { public: @@ -401,6 +501,930 @@ class json_sax_dom_parser return true; } + bool start_object(std::size_t len) + { +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS + // the values need their final place from the start (their parent + // pointers and positions refer to it), so no buffering + return start_stored_object(len); +#else + return start_object(len, std::integral_constant {}); +#endif + } + + bool key(string_t& val) + { + JSON_ASSERT(!frames.empty()); + return key(val, std::integral_constant {}); + } + + bool end_object() + { + JSON_ASSERT(!frames.empty()); + return end_object(std::integral_constant {}); + } + + bool start_array(std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > array_t().max_size())) + { + // the array does not exist yet: report with an empty one + BasicJsonType too_large(BasicJsonType::value_t::array); + return container_too_large_error(*this, len, "array", &too_large); + } + +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS + // the values need their final place from the start (their parent + // pointers and positions refer to it), so no buffering + start_stored_array(len); +#else + if (!collects(len)) + { + // the number of elements is known (or the elements must be in + // place as they arrive): store them directly + start_stored_array(len); + } + else + { + // the number of elements is not known, so growing the array + // would reallocate it (and, being a vector, move its elements) + // again and again. Collect the elements on a stack shared by all + // arrays instead, and copy them to an array of the exact size + // when the closing bracket arrives. + BasicJsonType* slot = nullptr; + std::size_t dest_member = 0; + container_destination(slot, dest_member); + reserve_elements(); + frames.push_back(frame{nullptr, nullptr, slot, elements.size(), ordered_state(dest_member, false)}); + in_object = false; + stored_array = nullptr; + } +#endif + return true; + } + + bool end_array() + { + JSON_ASSERT(!frames.empty()); + JSON_ASSERT(frames.back().object == nullptr); + const frame f = frames.back(); + frames.pop_back(); + reenter_enclosing_container(); + + if (f.array == nullptr) + { + // the elements are the top of the shared elements stack, from + // f.start on (the elements below belong to enclosing arrays that + // are still open): move them into the array and pop them + const auto first = elements.begin() + static_cast(f.start); + auto* const arr = BasicJsonType::template create(std::make_move_iterator(first), std::make_move_iterator(elements.end())); + BasicJsonType value; + value.m_data.m_value.array = arr; + value.m_data.m_type = BasicJsonType::value_t::array; + elements.erase(first, elements.end()); + + place_container(f, frames.empty() ? nullptr : &frames.back(), std::move(value), true); + return true; + } + +#if JSON_DIAGNOSTIC_POSITIONS + diagnostic_positions::set_container_end(*f.slot, m_lexer_ref); +#endif +#if JSON_DIAGNOSTICS + f.slot->set_parents(); +#endif + return true; + } + + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const Exception& ex) + { + errored = true; + static_cast(ex); + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + using object_t = typename BasicJsonType::object_t; + using array_t = typename BasicJsonType::array_t; + + /// whether objects keep their elements in a vector and a key can be hashed, + /// so that a big object can find the keys it has in a hash table instead of + /// searching all of them + template + using detect_indexable_keys = decltype(std::declval().emplace_new_key(std::declval(), std::declval()), + std::hash()(std::declval())); + static constexpr bool indexed_keys = is_detected::value; + + /// what a frame holds for objects that are kept in a vector: the key index + /// of an object with many keys, and for an object that is collected on the + /// stack the member of the enclosing object it will become + struct ordered_frame + { + /// the entry of key_indexes of an object that has one, else nullptr + object_key_index* index; + /// object or array without a slot in an object collected on the stack: + /// the index of the member in `members` it goes to + std::size_t dest_member; + /// whether this is an object collected on the stack + bool buffered; + }; + struct no_ordered_frame {}; + using ordered_frame_state = typename std::conditional::type; + + static ordered_frame make_ordered_state(const std::size_t dest_member, const bool buffered, std::true_type /*indexed*/) noexcept + { + ordered_frame state = {nullptr, dest_member, buffered}; + return state; + } + + static no_ordered_frame make_ordered_state(const std::size_t /*dest_member*/, const bool /*buffered*/, std::false_type /*indexed*/) noexcept + { + return no_ordered_frame(); + } + + static ordered_frame_state ordered_state(const std::size_t dest_member, const bool buffered) noexcept + { + return make_ordered_state(dest_member, buffered, std::integral_constant {}); + } + + /// an array or object that is being read + struct frame + { + /// the object, or nullptr for an array + object_t* object; + /// the array if its elements are stored in it as they arrive, or + /// nullptr for an array and an object + array_t* array; + /// array without `array`: where the finished array goes (nullptr: it + /// is an element of the enclosing array); otherwise: the value that + /// holds the object or array + BasicJsonType* slot; + /// array without `array`: the index of its first element in `elements` + std::size_t start; + /// what objects in a vector add (see ordered_frame) + ordered_frame_state ordered; + }; + + /// the open arrays and objects; the first few live in the object itself, so + /// that the common documents do not allocate for them + class frame_stack + { + public: + frame_stack() noexcept + : local() // NOLINT(readability-redundant-member-init) + {} + frame_stack(const frame_stack&) = delete; + frame_stack& operator=(const frame_stack&) = delete; + + frame_stack(frame_stack&& other) noexcept + : local() // NOLINT(readability-redundant-member-init) + { + take(other); + } + + frame_stack& operator=(frame_stack&& other) noexcept + { + if (this != &other) + { + release(); + take(other); + } + return *this; + } + + ~frame_stack() = default; + + bool empty() const noexcept + { + return count == 0; + } + + std::size_t size() const noexcept + { + return count; + } + + frame& back() noexcept + { + return data[count - 1]; + } + + frame& operator[](const std::size_t i) noexcept + { + return data[i]; + } + + void push_back(const frame& f) + { + if (JSON_HEDLEY_UNLIKELY(count == capacity)) + { + grow(); + } + data[count++] = f; + } + + void pop_back() noexcept + { + JSON_ASSERT(count > 0); + --count; + } + + private: + JSON_HEDLEY_NEVER_INLINE + void grow() + { + const std::size_t new_capacity = capacity * 2; + std::unique_ptr new_heap(new frame[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + std::copy(data, data + count, new_heap.get()); + heap = std::move(new_heap); + data = heap.get(); + capacity = new_capacity; + } + + void release() noexcept + { + heap.reset(); + data = local.data(); + capacity = local_capacity; + } + + /// make the contents of @a other our own + void take(frame_stack& other) noexcept + { + if (other.heap == nullptr) + { + std::copy(other.local.begin(), other.local.begin() + static_cast(other.count), local.begin()); + data = local.data(); + capacity = local_capacity; + } + else + { + heap = std::move(other.heap); + data = heap.get(); + capacity = other.capacity; + other.data = other.local.data(); + other.capacity = local_capacity; + } + count = other.count; + other.count = 0; + } + + static constexpr std::size_t local_capacity = 8; + + /// the first frames, left uninitialized: a frame is written before it is + /// read (the user-provided constructor keeps the member initialization, + /// which -Weffc++ asks for, from zeroing them) + struct local_frames : std::array + { + local_frames() noexcept {} // NOLINT(modernize-use-equals-default,hicpp-use-equals-default,cppcoreguidelines-pro-type-member-init,hicpp-member-init): see above + }; + + local_frames local; + /// the frames once there are more than local_capacity of them + std::unique_ptr heap {}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init) + frame* data = local.data(); + std::size_t count = 0; + std::size_t capacity = local_capacity; + }; + + void release_key_index(const frame& f, std::true_type /*indexed*/) + { + if (JSON_HEDLEY_UNLIKELY(f.ordered.index != nullptr)) + { + ordered_data.key_indexes.pop_back(); + } + } + + static void release_key_index(const frame& /*f*/, std::false_type /*indexed*/) noexcept {} + + /// the number of keys from which an object looks them up in a hash table + static constexpr std::size_t key_index_threshold = 64; + + /// add @a key to the object of @a f (or find it, if it is there), searching it in std::map + static BasicJsonType* insert_key(frame& f, string_t&& key, std::false_type /*indexed*/) + { + return &f.object->operator[](std::move(key)); + } + + /// add @a key to the object of @a f (or find it, if it is there), searching it in a vector + BasicJsonType* insert_key(frame& f, string_t&& key, std::true_type /*indexed*/) + { + object_t& obj = *f.object; + if (JSON_HEDLEY_LIKELY(obj.size() < key_index_threshold)) + { + return &obj[std::move(key)]; + } + return insert_key_with_index(f, std::move(key)); + } + + /// add @a key to the big object of @a f (or find it, if it is there), using its key index + JSON_HEDLEY_NEVER_INLINE + BasicJsonType* insert_key_with_index(frame& f, string_t&& key) + { + object_t& obj = *f.object; + const std::hash hasher {}; + const std::size_t hash = hasher(key); + if (JSON_HEDLEY_UNLIKELY(f.ordered.index == nullptr)) + { + std::vector hashes; + hashes.reserve(obj.size()); + for (const auto& element : obj) + { + hashes.push_back(hasher(element.first)); + } + ordered_data.key_indexes.emplace_back(new object_key_index(std::move(hashes))); + f.ordered.index = ordered_data.key_indexes.back().get(); + } + + const std::size_t position = f.ordered.index->find(hash, [&obj, &key](std::size_t i) + { + return (obj.begin() + static_cast(i))->first == key; + }); + if (position != object_key_index::npos) + { + return &(obj.begin() + static_cast(position))->second; + } + + obj.emplace_new_key(key, BasicJsonType()); + f.ordered.index->add(hash); + return &obj.back().second; + } + + /// where an array or object that starts now will go when it is finished: + /// @a slot is the value to assign it to (the root or a member of an object + /// that is stored directly), or nullptr if it is an element of the + /// enclosing array or a member of the enclosing object that is collected on + /// the stack, in which case @a dest_member is the member + void container_destination(BasicJsonType*& slot, std::size_t& dest_member) + { + if (frames.empty()) + { + slot = &root; + } + else if (frames.back().object != nullptr) + { + slot = object_element; + } + else + { + collected_object_member(frames.back(), dest_member, std::integral_constant {}); + } + } + + void collected_object_member(const frame& enclosing, std::size_t& dest_member, std::true_type /*indexed*/) const noexcept + { + if (enclosing.ordered.buffered) + { + dest_member = ordered_data.member_index; + } + } + + static void collected_object_member(const frame& /*enclosing*/, std::size_t& /*dest_member*/, std::false_type /*indexed*/) noexcept {} + + static bool is_collected_object(const frame& f, std::true_type /*indexed*/) noexcept + { + return f.ordered.buffered; + } + + static bool is_collected_object(const frame& /*f*/, std::false_type /*indexed*/) noexcept + { + return false; + } + + /// put a finished array or object that was collected on the stack, or + /// that outgrew it, where it belongs; @a enclosing is the container it is in. + /// @a enclosing_is_innermost tells whether that is the container that gets + /// the next value, so that it can be switched to storing its elements + /// directly if the value was the one too many. + void place_container(const frame& f, frame* enclosing, BasicJsonType&& value, const bool enclosing_is_innermost) + { + if (f.slot != nullptr) + { + *f.slot = std::move(value); + } + else if (is_collected_object(*enclosing, std::integral_constant {})) + { + place_member(f, std::move(value), std::integral_constant {}); + } + else if (enclosing->array != nullptr) + { + enclosing->array->emplace_back(std::move(value)); + } + else + { + elements.emplace_back(std::move(value)); + if (JSON_HEDLEY_UNLIKELY(enclosing_is_innermost && elements.size() - enclosing->start > buffer_limit)) + { + spill(*enclosing); + } + } + } + + void place_member(const frame& f, BasicJsonType&& value, std::true_type /*indexed*/) + { + ordered_data.members[f.ordered.dest_member].second = std::move(value); + } + + static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} + + /// store the members of a new object directly in it + bool start_stored_object(std::size_t len) + { + BasicJsonType* const ref = store(BasicJsonType::value_t::object); + frames.push_back(frame{ref->m_data.m_value.object, nullptr, ref, 0, ordered_state(0, false)}); + in_object = true; + stored_array = nullptr; + +#if JSON_DIAGNOSTIC_POSITIONS + // Ensure this is after the call to store to ensure correct start position. + diagnostic_positions::set_container_start(*ref, m_lexer_ref); +#endif + + return check_container_size(*this, len, "object", ref); + } + + /// objects in std::map and the like have no use for collecting their members + bool start_object(std::size_t len, std::false_type /*indexed*/) + { + return start_stored_object(len); + } + + /// An object in a vector of members adds a member by appending it to the + /// vector, which grows it again and again (and copies the keys each time). + /// Collect the members of an object of unknown size on a stack shared by + /// all open objects instead, and move them into a vector of the exact size + /// when the closing brace arrives. + bool start_object(std::size_t len, std::true_type /*indexed*/) + { + if (!collects(len)) + { + return start_stored_object(len); + } + + BasicJsonType* slot = nullptr; + std::size_t dest_member = 0; + container_destination(slot, dest_member); + frames.push_back(frame{nullptr, nullptr, slot, ordered_data.members.size(), ordered_state(dest_member, true)}); + in_object = false; + stored_array = nullptr; + return true; + } + + bool key(string_t& val, std::false_type /*indexed*/) + { + JSON_ASSERT(frames.back().object != nullptr); + + // add null at the given key and store the reference for later; the + // key is moved, as the lexer starts the next token with reset() + object_element = insert_key(frames.back(), std::move(val), std::false_type {}); + return true; + } + + bool key(string_t& val, std::true_type /*indexed*/) + { + frame& f = frames.back(); + if (JSON_HEDLEY_UNLIKELY(f.object == nullptr)) + { + JSON_ASSERT(f.ordered.buffered); + if (collect_member(f, val)) + { + return true; + } + // the object has grown big: it is stored directly from here on + } + object_element = insert_key(f, std::move(val), std::true_type {}); + return true; + } + + /*! + @brief add a member to the object that is collected on the stack, or find the one with the key + @return false if the object has collected enough members to be stored + directly, which is done and which leaves the key to be inserted + */ + bool collect_member(frame& f, string_t& key) + { + for (std::size_t i = f.start; i < ordered_data.members.size(); ++i) + { + if (ordered_data.members[i].first == key) + { + ordered_data.member_index = i; + return true; + } + } + + if (JSON_HEDLEY_UNLIKELY(ordered_data.members.size() - f.start >= object_buffer_limit)) + { + spill_object(f); + return false; + } + + reserve_members(); + ordered_data.members.emplace_back(std::move(key), BasicJsonType()); + ordered_data.member_index = ordered_data.members.size() - 1; + return true; + } + + /// the members of an object that is collected on the stack as an object; + /// they are the top of the shared member stack, from f.start on (the + /// members below belong to enclosing objects that are still open) + object_t* members_to_object(const frame& f, BasicJsonType& value) + { + const auto first = ordered_data.members.begin() + static_cast(f.start); + auto* const obj = BasicJsonType::template create(std::make_move_iterator(first), std::make_move_iterator(ordered_data.members.end())); + value.m_data.m_value.object = obj; + value.m_data.m_type = BasicJsonType::value_t::object; + ordered_data.members.erase(first, ordered_data.members.end()); + return obj; + } + + /// switch an object that collects its members on the stack to storing them directly + JSON_HEDLEY_NEVER_INLINE + void spill_object(frame& f) + { + BasicJsonType value; + f.object = members_to_object(f, value); + f.ordered.buffered = false; + in_object = true; + // false positive: an object without an enclosing container is the root, + // which has a slot, so place_container() does not use the nullptr + // @infer-ignore NULLPTR_DEREFERENCE + place_container(f, frames.size() >= 2 ? &frames[frames.size() - 2] : nullptr, std::move(value), false); + } + + bool end_object(std::false_type /*indexed*/) + { + return end_stored_object(); + } + + bool end_object(std::true_type /*indexed*/) + { + if (JSON_HEDLEY_LIKELY(frames.back().object != nullptr)) + { + return end_stored_object(); + } + + // the members are on the stack: make them the object + const frame f = frames.back(); + frames.pop_back(); + reenter_enclosing_container(); + + BasicJsonType value; + members_to_object(f, value); + place_container(f, frames.empty() ? nullptr : &frames.back(), std::move(value), true); + return true; + } + + bool end_stored_object() + { + JSON_ASSERT(frames.back().object != nullptr); + +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS + // objects are never collected, so their value is where it was stored + BasicJsonType* const ref = frames.back().slot; +#endif +#if JSON_DIAGNOSTIC_POSITIONS + diagnostic_positions::set_container_end(*ref, m_lexer_ref); +#endif +#if JSON_DIAGNOSTICS + ref->set_parents(); +#endif + + release_key_index(frames.back(), std::integral_constant {}); + frames.pop_back(); + reenter_enclosing_container(); + return true; + } + + /// the number of members an object collects on the stack before it + /// switches to storing them directly + static constexpr std::size_t object_buffer_limit = 32; + + void reserve_members() + { + if (JSON_HEDLEY_UNLIKELY(ordered_data.members.capacity() == 0)) + { + ordered_data.members.reserve(16); + } + } + + /// the number of elements an array collects on the stack before it + /// switches to storing them directly, so that it does not hold up the + /// stack with its elements (and move all of them in the end) + static constexpr std::size_t buffer_limit = 16; + + void reserve_elements() + { + if (JSON_HEDLEY_UNLIKELY(elements.capacity() == 0)) + { + elements.reserve(32); + } + } + + /// store the elements of a new array directly in it + void start_stored_array(const std::size_t len) + { + BasicJsonType* const ref = store(BasicJsonType::value_t::array); + frames.push_back(frame{nullptr, ref->m_data.m_value.array, ref, 0, ordered_state(0, false)}); + in_object = false; + stored_array = ref->m_data.m_value.array; + +#if JSON_DIAGNOSTIC_POSITIONS + // Ensure this is after the call to store to ensure correct start position. + diagnostic_positions::set_container_start(*ref, m_lexer_ref); +#endif + + if (len != detail::unknown_size()) + { + reserve_array(*ref->m_data.m_value.array, len, priority_tag<1> {}); + } + } + + /*! + @brief store a value as the next element of the open array or object (or + as the root) + @return the stored value + */ + template + JSON_HEDLEY_RETURNS_NON_NULL + BasicJsonType* store(Value&& v) + { + if (stored_array != nullptr) + { + stored_array->emplace_back(std::forward(v)); + return &stored_array->back(); + } + + if (in_object) + { + JSON_ASSERT(object_element); + set_member(*object_element, std::forward(v), std::is_same {}); + return object_element; + } + + return store_collected(std::forward(v), std::integral_constant {}); + } + + /// store a value that is the root, an element of an array that is collected + /// on the stack, or (in a vector) a member of an object that is + template + BasicJsonType* store_collected(Value&& v, std::true_type /*indexed*/) + { + if (!frames.empty() && frames.back().ordered.buffered) + { + BasicJsonType& member = ordered_data.members[ordered_data.member_index].second; + set_member(member, std::forward(v), std::is_same {}); + return &member; + } + return store_collected(std::forward(v), std::false_type {}); + } + + template + BasicJsonType* store_collected(Value&& v, std::false_type /*indexed*/) + { + if (frames.empty()) + { + root = BasicJsonType(std::forward(v)); + return &root; + } + + frame& f = frames.back(); + elements.emplace_back(std::forward(v)); + if (JSON_HEDLEY_UNLIKELY(elements.size() - f.start > buffer_limit)) + { + return spill(f); + } + return &elements.back(); + } + + /*! + @brief switch an array that collects its elements on the stack to storing + them directly + @return the last element of the array + */ + JSON_HEDLEY_NEVER_INLINE + BasicJsonType* spill(frame& f) + { + JSON_ASSERT(f.array == nullptr); + const auto first = elements.begin() + static_cast(f.start); + BasicJsonType value(BasicJsonType::value_t::array); + array_t& arr = *value.m_data.m_value.array; + // the capacity a vector that grew by itself has with this many elements + reserve_array(arr, 2 * buffer_limit, priority_tag<1> {}); + arr.insert(arr.end(), std::make_move_iterator(first), std::make_move_iterator(elements.end())); + elements.erase(first, elements.end()); + f.array = &arr; + stored_array = &arr; + + // the array takes the place its first element had on the stack + place_container(f, frames.size() >= 2 ? &frames[frames.size() - 2] : nullptr, std::move(value), false); + return &arr.back(); + } + + /// set the value of the member key() just added + template + JSON_HEDLEY_ALWAYS_INLINE + static void set_member(BasicJsonType& member, Value&& v, std::true_type /*default base*/) + { + if (JSON_HEDLEY_LIKELY(member.is_null())) + { + // a new member is null and holds nothing: construct the value + // over it instead of destroying and swapping + ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + } + else + { + // the key occurred before + member = BasicJsonType(std::forward(v)); + } + } + + /// a custom base class may care about the lifetime of the value + template + static void set_member(BasicJsonType& member, Value&& v, std::false_type /*custom base*/) + { + member = BasicJsonType(std::forward(v)); + } + + /// whether the values of a container of @a len elements are collected on the stack + static bool collects(const std::size_t len) noexcept + { + return collects(len, std::integral_constant {}); + } + + static bool collects(const std::size_t len, std::true_type /*collect*/) noexcept + { + return len == detail::unknown_size(); + } + + static bool collects(const std::size_t /*len*/, std::false_type /*collect*/) noexcept + { + return false; + } + + /// after a container was closed: the one it was in is the one that gets the next value + void reenter_enclosing_container() + { + if (frames.empty()) + { + in_object = false; + stored_array = nullptr; + } + else + { + const frame& f = frames.back(); + in_object = f.object != nullptr; + stored_array = f.array; + } + } + + template + void handle_value(Value&& v) + { +#if JSON_DIAGNOSTIC_POSITIONS + BasicJsonType* const stored = store(std::forward(v)); + diagnostic_positions::set_from_lexer(*stored, m_lexer_ref); +#else + store(std::forward(v)); +#endif + } + + /// the parsed JSON value + BasicJsonType& root; + /// the open arrays and objects + frame_stack frames {}; + /// the elements of the open arrays that are not stored in them yet + std::vector elements {}; + /// what objects in a vector need besides the frames (std::map based objects + /// need nothing, so they do not carry it) + struct ordered_data_t + { + /// the members of the open objects in a vector that are not stored in them yet + std::vector> members {}; + /// the member of the innermost such object that key() added or found last + std::size_t member_index = 0; + /// the key indexes of the open objects that have one, innermost last + std::vector> key_indexes {}; // NOLINT(readability-redundant-member-init) + }; + struct no_ordered_data_t {}; + typename std::conditional::type ordered_data {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether the innermost container is an object (frames.back().object) + bool in_object = false; + /// the array that stores the elements of the innermost array as they + /// arrive (frames.back().array), or nullptr + array_t* stored_array = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// the lexer reference to obtain the current position + lexer_t* m_lexer_ref = nullptr; +}; + +/*! +@brief SAX implementation to create a JSON value from the events of a binary reader + +The DOM parser of the text formats (json_sax_dom_parser) collects the elements +of arrays and the members of objects on a stack and builds them in one +exact-size allocation, which pays off because a text parser does not know the +number of elements up front. Binary formats announce the size of their +containers (CBOR, MessagePack, BSON and BON8 do, and so do the optimized +containers of UBJSON and BJData), so they store each value directly in its +container, and the bookkeeping of the collecting parser would be pure +overhead for them. This is the plain parser: the structure or hierarchy of the +JSON value is managed by the stack `ref_stack` which contains a pointer to the +respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template +class json_sax_dom_binary_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using lexer_t = lexer; + + /*! + @param[in,out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_binary_parser(BasicJsonType& r, const bool allow_exceptions_ = true, lexer_t* lexer_ = nullptr) + : root(r), allow_exceptions(allow_exceptions_), m_lexer_ref(lexer_) + {} + + // make class move-only + json_sax_dom_binary_parser(const json_sax_dom_binary_parser&) = delete; + json_sax_dom_binary_parser(json_sax_dom_binary_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + json_sax_dom_binary_parser& operator=(const json_sax_dom_binary_parser&) = delete; + json_sax_dom_binary_parser& operator=(json_sax_dom_binary_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~json_sax_dom_binary_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); + return true; + } + + bool binary(binary_t& val) + { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); + return true; + } + bool start_object(std::size_t len) { ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); @@ -663,16 +1687,13 @@ class json_sax_dom_callback_parser // handle_value() without touching the key stacks if (keep_stack.back()) { - BasicJsonType k = BasicJsonType(val); - static_cast(callback(static_cast(ref_stack.size()), parse_event_t::key, k)); + static_cast(callback(static_cast(ref_stack.size()), parse_event_t::key, key_value(val))); } return true; } - BasicJsonType k = BasicJsonType(val); - // check callback for the key - const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, key_value(val)); key_keep_stack.push_back(keep); // remember the key so a rejected value can be erased without searching // the object for it (kept in lockstep with key_keep_stack) @@ -683,7 +1704,7 @@ class json_sax_dom_callback_parser { auto& obj = *ref_stack.back()->m_data.m_value.object; // a single lookup: insert the placeholder, or find the existing member - const auto inserted = obj.emplace(val, discarded); + const auto inserted = obj.emplace(std::move(val), discarded); const auto it = inserted.first; if (!inserted.second) { @@ -857,6 +1878,31 @@ class json_sax_dom_callback_parser private: + /*! + @brief the value of a key event for the callback + + A string value of its own for every key would allocate it every time, so + one is kept and set to the key. A callback that changed or moved it gets a + new one, as it would have got a new value for every key anyway. + */ + BasicJsonType& key_value(const string_t& key) + { +#if JSON_DIAGNOSTIC_POSITIONS + // a callback may have left positions on the value + key_scratch = BasicJsonType(key); +#else + if (JSON_HEDLEY_LIKELY(key_scratch.is_string())) + { + *key_scratch.m_data.m_value.string = key; + } + else + { + key_scratch = BasicJsonType(key); + } +#endif + return key_scratch; + } + /// if there is a pending duplicate-key stash entry for this exact slot, /// remove it from the stash; if restore_value is true, the stashed /// previous value is moved back into the slot first (use this when the @@ -1104,6 +2150,8 @@ class json_sax_dom_callback_parser const bool allow_exceptions = true; /// a discarded value for the callback BasicJsonType discarded = BasicJsonType::value_t::discarded; + /// the value of the key event for the callback (see key_value()) + BasicJsonType key_scratch {}; /// the lexer reference to obtain the current position lexer_t* m_lexer_ref = nullptr; }; diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 8c13cd6d8..aec10c5d2 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -14,6 +14,7 @@ #include // snprintf #include // memcmp #include // initializer_list +#include // numeric_limits #include // char_traits, string #include // move #include // vector @@ -1816,6 +1817,15 @@ scan_number_done: /// significand (see scan_number_bulk_contiguous()) bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept { + // accept() only needs to know that the value is finite (see the parser): + // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a + // value that underflows is a finite number as well. Only a token that + // may overflow has to be converted to find out. + if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + { + value_float = 0; + return true; + } value_float = decimal_to_float(s); return true; } diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 38bc9cdc9..3017a29fd 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -56,8 +56,8 @@ using parser_callback_t = template struct sax_ignores_number_text : std::false_type {}; -template -struct sax_ignores_number_text> : std::true_type {}; +template +struct sax_ignores_number_text> : std::true_type {}; template struct sax_ignores_number_text> : std::true_type {}; @@ -131,7 +131,7 @@ class parser } else { - json_sax_dom_parser sdp(result, allow_exceptions, &m_lexer); + json_sax_dom_parser sdp(result, allow_exceptions, &m_lexer); // in case of an error, return a discarded value if (!parse_dom(sdp, strict)) @@ -243,12 +243,19 @@ class parser JSON_HEDLEY_NON_NULL(2) bool sax_parse_internal(SAX* sax) { + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; + // stack to remember the hierarchy of structured values we are parsing // true = array; false = object (bytes, not std::vector, whose // push_back() is a call with GCC) std::vector states; - // value to avoid a goto (see comment where set to true) - bool skip_to_state_evaluation = false; + // a document that starts with a container pushes onto the stack at + // once: allocate it in one step instead of growing it from 1 byte + if (last_token == token_type::begin_object || last_token == token_type::begin_array) + { + states.reserve(32); + } while (true) { diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index a382ae6ef..50fe7f53b 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -8,14 +8,14 @@ #pragma once -#include // all_of +#include // all_of, copy #include // array #include // isdigit #include // errno, ERANGE #include // size_t #include // uint64_t #include // strtoull -#include // memcmp, memcpy +#include // memchr, memcmp, memcpy #ifndef JSON_NO_IO #include // ostream #endif // JSON_NO_IO @@ -80,14 +80,38 @@ class json_pointer /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ string_t to_string() const { + if (reference_tokens.empty()) + { + return string_t(); + } + // one buffer: every token adds its escaped form (at least its length) and a slash std::size_t length = 0; + bool plain = true; for (const auto& reference_token : reference_tokens) { length += reference_token.size() + 1; + const char* const data = reference_token.data(); + for (std::size_t i = 0, size = reference_token.size(); plain && i < size; ++i) + { + plain = data[i] != '~' && data[i] != '/'; + } } string_t result; + if (JSON_HEDLEY_LIKELY(plain)) + { + // nothing to escape: the tokens are copied into the string + result.resize(length); + char* out = &result[0]; + for (const auto& reference_token : reference_tokens) + { + *out++ = '/'; + out = std::copy(reference_token.data(), reference_token.data() + reference_token.size(), out); + } + return result; + } + result.reserve(length); for (const auto& reference_token : reference_tokens) { @@ -303,6 +327,12 @@ class json_pointer exceeds_size_type ///< @a s converts to an integer that exceeds size_type }; + /// whether a reference token is "-" (a plain comparison with a string literal is not always inlined) + static bool is_dash(const string_t& s) noexcept + { + return s.size() == 1 && s[0] == '-'; + } + /*! @param[in] s reference token to be converted into an array index @param[out] idx the integer representation of @a s if @ref array_index_status::ok @@ -328,7 +358,7 @@ class json_pointer // error condition (cf. RFC 6901, Sect. 4); this also covers single- // character tokens, so "/x" and "/xy" fail alike; "-" and the empty // token are left to the conversion below and are reported as unresolved - if (JSON_HEDLEY_UNLIKELY(!s.empty() && s != "-" && !(s[0] >= '0' && s[0] <= '9'))) + if (JSON_HEDLEY_UNLIKELY(!s.empty() && !is_dash(s) && !(s[0] >= '0' && s[0] <= '9'))) { return array_index_status::not_a_number; } @@ -475,9 +505,40 @@ class json_pointer return pos == size; } + /// reserve room for @a n elements of an array type that has reserve() + template + static auto reserve_elements(ArrayType& array, const std::size_t n, detail::priority_tag<1> /*unused*/) + -> decltype(array.reserve(n), void()) + { + array.reserve(n); + } + + template + static void reserve_elements(ArrayType& /*array*/, const std::size_t /*n*/, detail::priority_tag<0> /*unused*/) + {} + /// @brief object member access with a key that is only copied if it is inserted + /// + /// Objects that can find the position of a key and insert there + /// (std::map) search once, others look the key up and add it if it is missing. template - static BasicJsonType& object_member(BasicJsonType& object, const string_t& key) + static auto object_member_impl(BasicJsonType& object, const string_t& key, detail::priority_tag<1> /*unused*/) + -> decltype(object.m_data.m_value.object->emplace_hint(object.m_data.m_value.object->lower_bound(key), key, nullptr), std::declval()) + { + auto& members = *object.m_data.m_value.object; + auto it = members.lower_bound(key); + if (it != members.end() && !members.key_comp()(key, it->first)) + { + return it->second; + } + + const auto old_capacity = object.container_capacity(); + it = members.emplace_hint(it, key, nullptr); + return object.set_parent_after_insert(it->second, old_capacity); + } + + template + static BasicJsonType& object_member_impl(BasicJsonType& object, const string_t& key, detail::priority_tag<0> /*unused*/) { auto& members = *object.m_data.m_value.object; const auto it = members.find(key); @@ -488,6 +549,12 @@ class json_pointer return object.operator[](key); } + template + static BasicJsonType& object_member(BasicJsonType& object, const string_t& key) + { + return object_member_impl(object, key, detail::priority_tag<1> {}); + } + JSON_PRIVATE_UNLESS_TESTED: json_pointer top() const { @@ -720,48 +787,103 @@ class json_pointer } } + /*! + @brief the values along the pointer last resolved by @ref resolve_plain + + A patch from @ref basic_json::diff lists its operations in document order, + so consecutive pointers share most of their reference tokens; the values + along the previous pointer are kept to resume from the common prefix. + `steps[k]` is the value after the first `k` reference tokens together with + the length of the prefix of `*path` that holds them. A value stays valid as + long as no operation changes its container: callers keep only values up to + the parent of the pointer they modify, or call `reset()`. + */ + template + struct plain_steps + { + const string_t* path = nullptr; + std::vector> steps{}; + + void reset() noexcept + { + path = nullptr; + steps.clear(); + } + }; + /*! @brief resolve a plain JSON pointer without building its reference tokens - @param[in] ptr the value the pointer starts at + @param[in] root the value the pointer starts at @param[in] path a JSON pointer string @param[in] length the number of leading bytes of @a path to resolve; 0 or the position of a slash @param[in,out] token a string to hold the reference tokens, to reuse its storage + @param[in,out] cache the values along the previously resolved pointer; + @a path must outlive the next call @return the pointed to value, or `nullptr` if the pointer does not denote - an existing value, or does so in a way that is not plain: a - reference token with an escape, an array index that is malformed or - out of range, or a "-". No exception is thrown for these; callers - use @ref json_pointer instead, which handles them. + an existing value, or does so in a way that is not plain: an array + index that is malformed or out of range, or a "-", or a reference + token with an escape. No exception is + thrown for these; callers use @ref json_pointer instead, which + handles them. */ template - static BasicJsonType* resolve_plain(BasicJsonType* ptr, const string_t& path, const std::size_t length, string_t& token) + static BasicJsonType* resolve_plain(BasicJsonType* root, const string_t& path, const std::size_t length, string_t& token, plain_steps& cache) { - for (std::size_t pos = 0; pos < length;) + auto& steps = cache.steps; + + // resume from the deepest step that is also a prefix of this path + std::size_t depth = 1; + if (cache.path != nullptr && !steps.empty() && steps[0].second == root) + { + const string_t& previous = *cache.path; + for (depth = steps.size(); depth > 1; --depth) + { + // a step ends at a token boundary of the previous path + const std::size_t prefix = steps[depth - 1].first; + if (prefix <= length && (prefix == length || path[prefix] == '/') && std::memcmp(path.data(), previous.data(), prefix) == 0) + { + break; + } + } + steps.resize(depth); + } + else + { + steps.assign(1, std::make_pair(std::size_t{0}, root)); + } + cache.path = &path; + + BasicJsonType* ptr = steps[depth - 1].second; + for (std::size_t pos = steps[depth - 1].first; pos < length;) { if (JSON_HEDLEY_UNLIKELY(path[pos] != '/')) { return nullptr; } - // the reference token between the slash at pos and the next one - std::size_t end = path.find_first_of('/', pos + 1); - if (end == string_t::npos || end > length) + // the reference token between the slash at pos and the next one, + // which must not have an escape + const char* const begin = path.data() + pos + 1; + const char* const limit = path.data() + length; + const char* end = begin; + while (end < limit && *end != '/') { - end = length; - } - token.clear(); - token.append(path.data() + pos + 1, end - pos - 1); - if (JSON_HEDLEY_UNLIKELY(token.find_first_of('~') != string_t::npos)) - { - return nullptr; + if (JSON_HEDLEY_UNLIKELY(*end == '~')) + { + return nullptr; + } + ++end; } switch (ptr->type()) { case detail::value_t::object: { + token.clear(); + token.append(begin, static_cast(end - begin)); const auto it = ptr->m_data.m_value.object->find(token); if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) { @@ -773,8 +895,24 @@ class json_pointer case detail::value_t::array: { - typename BasicJsonType::size_type idx{}; - if (JSON_HEDLEY_UNLIKELY(parse_array_index(token, idx) != array_index_status::ok || idx >= ptr->m_data.m_value.array->size())) + // an index of up to 18 digits without a leading zero fits into size_type + // (9 digits for a 32-bit size_type); everything else is left to json_pointer + constexpr std::size_t max_digits = sizeof(typename BasicJsonType::size_type) >= 8 ? 18 : 9; + const auto digits = static_cast(end - begin); + if (JSON_HEDLEY_UNLIKELY(digits == 0 || digits > max_digits || (digits > 1 && *begin == '0'))) + { + return nullptr; + } + typename BasicJsonType::size_type idx = 0; + for (const char* c = begin; c != end; ++c) + { + if (JSON_HEDLEY_UNLIKELY(*c < '0' || *c > '9')) + { + return nullptr; + } + idx = static_cast((idx * 10) + static_cast(*c - '0')); + } + if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size())) { return nullptr; } @@ -793,7 +931,8 @@ class json_pointer default: return nullptr; } - pos = end; + pos = static_cast(end - path.data()); + steps.emplace_back(pos, ptr); } return ptr; @@ -907,7 +1046,7 @@ class json_pointer case detail::value_t::array: { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + if (JSON_HEDLEY_UNLIKELY(is_dash(reference_token))) { // "-" always fails the range check JSON_THROW(detail::out_of_range::create(detail::exception_id::pointer_past_the_end_index, detail::concat( @@ -978,7 +1117,7 @@ class json_pointer case detail::value_t::array: { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + if (JSON_HEDLEY_UNLIKELY(is_dash(reference_token))) { // "-" cannot be used for const access JSON_THROW(detail::out_of_range::create(detail::exception_id::pointer_past_the_end_index, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr)); @@ -1040,7 +1179,7 @@ class json_pointer case detail::value_t::array: { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + if (JSON_HEDLEY_UNLIKELY(is_dash(reference_token))) { // "-" always fails the range check return nullptr; @@ -1117,7 +1256,7 @@ class json_pointer case detail::value_t::array: { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + if (JSON_HEDLEY_UNLIKELY(is_dash(reference_token))) { // "-" always fails the range check return false; @@ -1198,33 +1337,32 @@ class json_pointer } result.reserve(token_count); - // extract the reference tokens: - // - slash: position of the last read slash (or end of string) - // - start: position after the previous slash - for ( - // search for the first slash after the first character - std::size_t slash = reference_string.find_first_of('/', 1), - // set the beginning of the first reference token - start = 1; - // we can stop if start == 0 (if slash == string_t::npos) - start != 0; - // set the beginning of the next reference token - // (will eventually be 0 if slash == string_t::npos) - start = (slash == string_t::npos) ? 0 : slash + 1, - // find next slash - slash = reference_string.find_first_of('/', start)) + // extract the reference tokens: every one runs from after a slash to + // the next slash or the end of the string; a token without a '~' (the + // common case) is stored as it is, others are checked and unescaped + const char* const data = reference_string.data(); + const char* const end = data + size; + for (const char* start = data + 1;; ++start) { - // use the text between the beginning of the reference token - // (start) and the last slash (slash). - const auto count = (slash == string_t::npos ? reference_string.size() : slash) - start; - auto reference_token = string_t(reference_string.data() + start, count); - - // check reference tokens are properly escaped; a token without - // a '~' (the common case) needs neither check nor unescaping - std::size_t pos = reference_token.find_first_of('~'); - if (pos != string_t::npos) + const char* slash = start; + bool escaped = false; + while (slash != end && *slash != '/') { - for (; pos != string_t::npos; pos = reference_token.find_first_of('~', pos + 1)) + escaped = escaped || *slash == '~'; + ++slash; + } + + const auto count = static_cast(slash - start); + if (JSON_HEDLEY_LIKELY(!escaped)) + { + result.emplace_back(start, count); + } + else + { + auto reference_token = string_t(start, count); + + // check reference tokens are properly escaped + for (std::size_t pos = reference_token.find_first_of('~'); pos != string_t::npos; pos = reference_token.find_first_of('~', pos + 1)) { JSON_ASSERT(reference_token[pos] == '~'); @@ -1238,10 +1376,14 @@ class json_pointer } detail::unescape(reference_token); + result.push_back(std::move(reference_token)); } - // finally, store the reference token - result.push_back(std::move(reference_token)); + if (slash == end) + { + break; + } + start = slash; } return result; @@ -1276,6 +1418,52 @@ class json_pointer obj[key] = std::forward(v); } + /// the number of entries that flattening @a value adds: its primitive + /// values and empty containers + template + static std::size_t count_flattened(const BasicJsonType& value) + { + std::size_t count = 0; + std::vector pending(1, &value); + while (!pending.empty()) + { + const BasicJsonType* const v = pending.back(); + pending.pop_back(); + if (v->is_array() && !v->m_data.m_value.array->empty()) + { + for (const auto& element : *v->m_data.m_value.array) + { + pending.push_back(&element); + } + } + else if (v->is_object() && !v->m_data.m_value.object->empty()) + { + for (const auto& member : *v->m_data.m_value.object) + { + pending.push_back(&member.second); + } + } + else + { + ++count; + } + } + return count; + } + + // Reserve the storage for @a value of objects that have a reserve(), so + // that it does not have to be moved while it grows + template + static auto reserve_flattened(ObjectType& obj, const BasicJsonType& value, detail::priority_tag<1> /*unused*/) + -> decltype(obj.reserve(std::size_t {}), void()) + { + obj.reserve(count_flattened(value)); + } + + template + static void reserve_flattened(ObjectType& /*obj*/, const BasicJsonType& /*value*/, detail::priority_tag<0> /*unused*/) + {} + /// append the decimal representation of @a i to @a out static void append_index(string_t& out, std::size_t i) { @@ -1385,6 +1573,7 @@ class json_pointer } }; + reserve_flattened(object, value, detail::priority_tag<1> {}); enter(value); while (!stack.empty()) { @@ -1392,8 +1581,9 @@ class json_pointer // frame, which would invalidate a reference to it const BasicJsonType* const container = stack.back().container; - // drop the path of the previous child - path.resize(stack.back().path_length); + // the path of the previous child is replaced by the path of the + // next one; for an array element in one resize + const std::size_t path_length = stack.back().path_length; if (container->is_array()) { @@ -1407,8 +1597,20 @@ class json_pointer // iterate array and use index as a reference string ++stack.back().index; - path.push_back('/'); - append_index(path, i); + std::array digits{}; + std::size_t pos = digits.size(); + std::size_t rest = i; + do + { + digits[--pos] = static_cast('0' + (rest % 10)); + rest /= 10; + } + while (rest != 0); + const std::size_t count = digits.size() - pos; + path.resize(path_length + 1 + count); + char* const out = &path[path_length]; + out[0] = '/'; + std::memcpy(out + 1, digits.data() + pos, count); enter(array[i]); } else @@ -1422,6 +1624,7 @@ class json_pointer // iterate object and use keys as reference string ++stack.back().member; + path.resize(path_length); path.push_back('/'); detail::escape_into(path, it->first); enter(it->second); @@ -1517,8 +1720,28 @@ class json_pointer { end = key.size(); } + const std::size_t token_size = end - pos - 1; + + // a token that is a plain index of up to 9 digits, which is + // the case for almost all array elements, is not copied + std::size_t index = 0; + bool plain_index = current->type() == detail::value_t::array && token_size != 0 && token_size <= 9 && (token_size == 1 || data[pos + 1] != '0'); + for (std::size_t k = pos + 1; plain_index && k < end; ++k) + { + plain_index = data[k] >= '0' && data[k] <= '9'; + index = (index * 10) + static_cast(data[k] - '0'); + } + if (plain_index) + { + current = ¤t->operator[](static_cast(index)); + values.push_back(current); + ends.push_back(end); + pos = end; + continue; + } + token.clear(); - token.append(data + pos + 1, end - pos - 1); + token.append(data + pos + 1, token_size); if (token.find_first_of('~') != string_t::npos) { detail::unescape(token); @@ -1528,10 +1751,13 @@ class json_pointer { case detail::value_t::null: { - if (arrays.contains(data, pos)) + // a first child 0 makes the value an array without a look-up + if ((token_size == 1 && data[pos + 1] == '0') || arrays.contains(data, pos)) { // some reference token below this position is 0, so the // value is an array + *current = BasicJsonType(detail::value_t::array); + reserve_elements(*current->m_data.m_value.array, 2, detail::priority_tag<1> {}); current = ¤t->operator[](array_index(token)); } else diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 916307b89..f75c84a48 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -8,7 +8,7 @@ #pragma once -#include // reverse +#include // max, min, reverse #include // array #include // isnan, isinf #include // uint8_t, uint16_t, uint32_t, uint64_t @@ -75,6 +75,21 @@ std::size_t binary_reserve_hint(const BasicJsonType& j) return 0; } +/*! +@brief the capacity to reserve for the result of a binary serialization into a + std::vector + +The hint of @ref binary_reserve_hint, but not less than the 64 characters that +output_vector_sink grows an empty vector to on its first write anyway. A small +document then costs one allocation, instead of an allocation of a few bytes that +the first write replaces with one of 64. +*/ +template +std::size_t binary_initial_capacity(const BasicJsonType& j) +{ + return (std::max)(binary_reserve_hint(j), min_vector_sink_capacity); +} + /*! @brief serialization to BJData, BON8, BSON, CBOR, MessagePack, and UBJSON values */ @@ -123,9 +138,22 @@ class binary_writer template < typename SinkType = OutputSinkType, typename std::enable_if < std::is_constructible>::value, int >::type = 0 > explicit binary_writer(output_adapter_t adapter, const error_handler_t error_handler_ = binary_writer_default_error_handler()) - : oa(SinkType(std::move(adapter))), error_handler(error_handler_) + : oa(std::move(adapter)), error_handler(error_handler_) {} + /*! + @brief pass on everything written so far to the output adapter + + A writer over an output_adapter_sink buffers what it writes; this has to + be called after the last write, or the buffered end is only passed on, and + an error of the adapter lost, when the writer is destroyed. A writer over + an output_vector_sink writes straight into the vector. + */ + void flush() + { + oa.flush(); + } + /*! @param[in] j JSON value to serialize @throw type_error.316 if a string value or an object key is not valid @@ -292,16 +320,7 @@ class binary_writer case value_t::object: { - // step 1: write control byte and the object size - write_cbor_head(0xA0, j.m_data.m_value.object->size()); - - for (const auto& el : *j.m_data.m_value.object) - { - // el.first is written directly (not via a temporary - // basic_json), with the object as diagnostics context - write_cbor_key(el.first, j); - write_cbor(el.second, depth + 1); - } + write_cbor_object(j, depth); break; } @@ -321,7 +340,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(length))) { - JSON_THROW(out_of_range::create(exception_id::length_too_large, concat("MessagePack length ", std::to_string(length), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), &j)); + throw_msgpack_length_too_large(length, j); } static_cast(j); @@ -410,7 +429,7 @@ class binary_writer case value_t::number_float: { - write_compact_float(j.m_data.m_value.number_float, to_char_type(0xCA), to_char_type(0xCB)); + write_msgpack_float(j.m_data.m_value.number_float); break; } @@ -500,7 +519,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(j.m_data.m_value.binary->subtype() > (std::numeric_limits::max)())) { - JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(j.m_data.m_value.binary->subtype()), " is too large for the MessagePack ext type (max 255)"), &j)); + throw_subtype_out_of_range(j.m_data.m_value.binary->subtype(), "the MessagePack ext type", j); } write_number(static_cast(j.m_data.m_value.binary->subtype())); @@ -517,15 +536,7 @@ class binary_writer case value_t::object: { // step 1: write control byte and the object size - write_msgpack_object_prefix(j.m_data.m_value.object->size(), j); - - for (const auto& el : *j.m_data.m_value.object) - { - // as in write_cbor, el.first is written directly with the - // object as diagnostics context - write_msgpack_key(el.first, j); - write_msgpack(el.second, depth + 1); - } + write_msgpack_object(j, depth); break; } @@ -587,14 +598,16 @@ class binary_writer } case value_t::number_integer: - { - write_number_with_ubjson_prefix(j.m_data.m_value.number_integer, add_prefix, use_bjdata); - break; - } - case value_t::number_unsigned: { - write_number_with_ubjson_prefix(j.m_data.m_value.number_unsigned, add_prefix, use_bjdata); + if (j.m_data.m_type == value_t::number_integer) + { + write_number_with_ubjson_prefix(j.m_data.m_value.number_integer, add_prefix, use_bjdata); + } + else + { + write_number_with_ubjson_prefix(j.m_data.m_value.number_unsigned, add_prefix, use_bjdata); + } break; } @@ -606,17 +619,7 @@ class binary_writer case value_t::string: { - string_t storage; - const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage); - - if (add_prefix) - { - oa.write_character(to_char_type('S')); - } - write_number_with_ubjson_prefix(value.size(), true, use_bjdata); - oa.write_characters( - reinterpret_cast(value.data()), - value.size()); + write_ubjson_text(*j.m_data.m_value.string, j, add_prefix, use_bjdata); break; } @@ -625,9 +628,19 @@ class binary_writer bool prefix_required = true; const bool write_closer = write_ubjson_start_array(j, use_count, use_type, add_prefix, use_bjdata, prefix_required); - for (const auto& el : *j.m_data.m_value.array) { - write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + const auto& array = *j.m_data.m_value.array; + if (starts_with_ubjson_number(array, use_bjdata, std::integral_constant())) + { + write_ubjson_array_elements(array, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth); + } + else + { + for (const auto& el : array) + { + write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + } + } } if (write_closer) @@ -649,7 +662,7 @@ class binary_writer { if (!use_count) { - JSON_THROW(other_error::create(exception_id::size_marker_required, "use_type requires use_size = true", &j)); + throw_size_marker_required(j); } oa.write_character(to_char_type('$')); oa.write_character(bjdata_draft3 ? 'B' : 'U'); @@ -669,13 +682,7 @@ class binary_writer } else { - for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) - { - oa.write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); - // the cast is needed for binary types whose value type - // is not an integer (e.g., std::byte) - oa.write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); - } + write_ubjson_binary_bytes(*j.m_data.m_value.binary, bjdata_draft3); } if (!use_count) @@ -699,18 +706,7 @@ class binary_writer bool prefix_required = true; const bool write_closer = write_ubjson_start_object(j, use_count, use_type, add_prefix, use_bjdata, prefix_required); - for (const auto& el : *j.m_data.m_value.object) - { - // a converted key must outlive the reference returned by sanitize_utf8_for_write - const object_key_string_t key_string = el.first; - string_t storage; - const string_t& key = sanitize_utf8_for_write(key_string, j, storage); - write_number_with_ubjson_prefix(key.size(), true, use_bjdata); - oa.write_characters( - reinterpret_cast(key.data()), - key.size()); - write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); - } + write_ubjson_members(j, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth); if (write_closer) { @@ -875,7 +871,8 @@ class binary_writer @brief throws because the integer @a j is too large for @a format_name @throw out_of_range.407 always */ - JSON_HEDLEY_NO_RETURN static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) { // neither std::to_string(), which has no overload for integer types // wider than 64 bits, nor dump(), which would also instantiate the @@ -891,13 +888,53 @@ class binary_writer JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", number, " cannot be represented by ", format_name, " as it does not fit ", range), &j)); } + // The exceptions below are thrown from functions of their own, out of line: + // the code that builds a message (concat, std::to_string) would otherwise + // make the frame of the function that checks, which most calls never throw. + + /// @throw out_of_range.412 always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_msgpack_length_too_large(const std::size_t length, const BasicJsonType& j) + { + static_cast(length); // unused when JSON_NOEXCEPTION is defined + static_cast(j); + JSON_THROW(out_of_range::create(exception_id::length_too_large, concat("MessagePack length ", std::to_string(length), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), &j)); + } + + /// @throw out_of_range.412 always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_bson_length_too_large(const std::size_t size) + { + static_cast(size); // unused when JSON_NOEXCEPTION is defined + JSON_THROW(out_of_range::create(exception_id::length_too_large, concat("BSON length ", std::to_string(size), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), nullptr)); + } + + /// @throw out_of_range.415 always + template + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_subtype_out_of_range(const SubtypeType subtype, const char* what, const BasicJsonType& j) + { + static_cast(subtype); // unused when JSON_NOEXCEPTION is defined + static_cast(what); + static_cast(j); + JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(subtype), " is too large for ", what, " (max 255)"), &j)); + } + + /// @throw other_error.407 always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_size_marker_required(const BasicJsonType& j) + { + static_cast(j); // unused when JSON_NOEXCEPTION is defined + JSON_THROW(other_error::create(exception_id::size_marker_required, "use_type requires use_size = true", &j)); + } + void write_msgpack_array_prefix(const std::size_t N, const BasicJsonType& j) { const auto n = to_msgpack_length(N, j); if (n <= 15) { // fixarray - write_number(static_cast(0x90 | n)); + oa.write_character(to_char_type(static_cast(0x90 | n))); } else if (n <= (std::numeric_limits::max)()) { @@ -919,7 +956,7 @@ class binary_writer if (n <= 15) { // fixmap - write_number(static_cast(0x80 | (n & 0xF))); + oa.write_character(to_char_type(static_cast(0x80 | (n & 0xF)))); } else if (n <= (std::numeric_limits::max)()) { @@ -1148,7 +1185,7 @@ class binary_writer { if (!use_count) { - JSON_THROW(other_error::create(exception_id::size_marker_required, "use_type requires use_size = true", &j)); + throw_size_marker_required(j); } const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); const bool same_prefix = std::all_of(j.begin() + 1, j.end(), @@ -1177,8 +1214,7 @@ class binary_writer if (use_count) { - oa.write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); + write_ubjson_count(j.m_data.m_value.array->size(), use_bjdata); } return !use_count; @@ -1198,7 +1234,7 @@ class binary_writer { if (!use_count) { - JSON_THROW(other_error::create(exception_id::size_marker_required, "use_type requires use_size = true", &j)); + throw_size_marker_required(j); } const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); const bool same_prefix = std::all_of(j.begin(), j.end(), @@ -1217,8 +1253,7 @@ class binary_writer if (use_count) { - oa.write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); + write_ubjson_count(j.m_data.m_value.object->size(), use_bjdata); } return !use_count; @@ -1384,12 +1419,7 @@ class binary_writer // a converted key must outlive the reference returned by sanitize_utf8_for_write const object_key_string_t key_string = current.object_it->first; - string_t storage; - const string_t& key = sanitize_utf8_for_write(key_string, j, storage); - write_number_with_ubjson_prefix(key.size(), true, use_bjdata); - oa.write_characters( - reinterpret_cast(key.data()), - key.size()); + write_ubjson_text(key_string, j, false, use_bjdata); const BasicJsonType* child = &(current.object_it->second); const bool child_prefix = current.prefix_required; ++stack.back().object_it; @@ -1412,18 +1442,85 @@ class binary_writer */ std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) { - const auto it = name.find(static_cast(0)); + const std::size_t it = find_nul(name); if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos)) { - JSON_THROW(out_of_range::create(exception_id::bson_key_with_null, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j)); + throw_bson_key_with_null(it, j); } + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + return calc_bson_checked_entry_header_size(name, j); + } + + /// @brief throw out_of_range.409 for a name with U+0000 at byte @a index; + /// out of line, so that the message does not burden the check + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_bson_key_with_null(const std::size_t index, const BasicJsonType& j) + { + static_cast(index); // only used when exceptions are enabled + static_cast(j); + JSON_THROW(out_of_range::create(exception_id::bson_key_with_null, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(index), ")"), &j)); + } + + /// @brief the size of a BSON document entry header for a name that the + /// error handler checks or sanitizes; out of line, as the string_t + /// for the sanitized copy would otherwise burden every call of + /// @ref calc_bson_entry_header_size + JSON_HEDLEY_NEVER_INLINE + std::size_t calc_bson_checked_entry_header_size(const string_t& name, const BasicJsonType& j) + { string_t storage; const string_t& sanitized = sanitize_utf8_for_write(name, j, storage); return /*id*/ 1ul + sanitized.size() + /*zero-terminator*/1u; } + /*! + @brief the position of the first U+0000 in @a name, or string_t::npos + + The same as `name.find(0)`, without the calls into the standard library for + names of a few characters: it looks at eight characters at a time. + */ + static std::size_t find_nul(const string_t& name) noexcept + { + return find_nul(name, std::integral_constant()); + } + + static std::size_t find_nul(const string_t& name, std::true_type /*is_byte*/) noexcept + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + const std::size_t size = name.size(); + const char* const data = reinterpret_cast(name.data()); + std::size_t pos = 0; + for (; pos + 8 <= size; pos += 8) + { + std::uint64_t word = 0; + std::memcpy(&word, data + pos, sizeof(word)); + if (((word - ones) & ~word & high) != 0) + { + break; + } + } + for (; pos < size; ++pos) + { + if (data[pos] == '\0') + { + return pos; + } + } + return BasicJsonType::string_t::npos; + } + + static std::size_t find_nul(const string_t& name, std::false_type /*is_byte*/) noexcept + { + return name.find(static_cast(0)); + } + /*! @brief Checks that @a size fits into the 32-bit length field used by BSON @return The size as a signed 32-bit integer @@ -1433,7 +1530,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) { - JSON_THROW(out_of_range::create(exception_id::length_too_large, concat("BSON length ", std::to_string(size), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), nullptr)); + throw_bson_length_too_large(size); } return static_cast(size); @@ -1503,13 +1600,25 @@ class binary_writer { if (JSON_HEDLEY_LIKELY(value_in_range_of(value.size()))) { - string_t storage; - const string_t& sanitized = sanitize_utf8_for_write(value, j, storage); - return sizeof(std::int32_t) + sanitized.size() + 1ul; + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + return calc_bson_checked_string_size(value, j); } return sizeof(std::int32_t) + value.size() + 1ul; } + /// @brief the size of a BSON string that the error handler checks or + /// sanitizes; out of line, like @ref calc_bson_checked_entry_header_size + JSON_HEDLEY_NEVER_INLINE + std::size_t calc_bson_checked_string_size(const string_t& value, const BasicJsonType& j) + { + string_t storage; + const string_t& sanitized = sanitize_utf8_for_write(value, j, storage); + return sizeof(std::int32_t) + sanitized.size() + 1ul; + } + /*! @brief Writes a BSON element with key @a name and string value @a value @@ -1619,7 +1728,7 @@ class binary_writer if (value.has_subtype() && JSON_HEDLEY_UNLIKELY(value.subtype() > (std::numeric_limits::max)())) { - JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(value.subtype()), " is too large for the BSON binary subtype (max 255)"), &j)); + throw_subtype_out_of_range(value.subtype(), "the BSON binary subtype", j); } return sizeof(std::int32_t) + value.size() + 1ul; @@ -1699,12 +1808,198 @@ class binary_writer } } + /// the longest name for which write_bson_scalar() writes an entry through a + /// single reservation: the type, name, terminating null byte, and the length + /// of a string or binary value (4 bytes) or the largest fixed-size value (8 + /// bytes, a double or a 64-bit integer) must fit max_reserve_size + static constexpr std::size_t bson_scalar_max_name_size = max_reserve_size - 2 - 8 - 1; + + /*! + @brief store @a n at @a out in little-endian byte order + */ + template + void store_little_endian(CharType* out, const NumberType n) const noexcept + { + std::memcpy(out, &n, sizeof(NumberType)); + if (!is_little_endian) + { + std::reverse(out, out + sizeof(NumberType)); + } + } + + /*! + @brief encode the type and the value of the BSON document entry for @a j + if it has a fixed size + @param[in] out where the type byte goes + @param[in] value_out where the value goes: the characters after the name + and its terminating null byte + @return the end of the entry, or nullptr (and nothing is written to + @a out) if @a j is not a boolean, number, or null; at most 9 + characters are written to @a out and @a value_out + + The entry is encoded as @ref write_bson_value does. + */ + CharType* encode_bson_fixed_entry(CharType* out, CharType* value_out, const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::boolean: + out[0] = to_char_type(0x08); + *value_out++ = j.m_data.m_value.boolean ? to_char_type(0x01) : to_char_type(0x00); + return value_out; + + case value_t::number_float: + out[0] = to_char_type(0x01); + store_little_endian(value_out, static_cast(j.m_data.m_value.number_float)); + return value_out + sizeof(double); + + case value_t::number_integer: + { + const auto value = static_cast(j.m_data.m_value.number_integer); + if ((std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)()) + { + out[0] = to_char_type(0x10); // int32 + store_little_endian(value_out, static_cast(value)); + return value_out + sizeof(std::int32_t); + } + out[0] = to_char_type(0x12); // int64 + store_little_endian(value_out, value); + return value_out + sizeof(std::int64_t); + } + + case value_t::number_unsigned: + { + const auto value = conditional_static_cast(j.m_data.m_value.number_unsigned); + if (value <= static_cast((std::numeric_limits::max)())) + { + out[0] = to_char_type(0x10); // int32 + store_little_endian(value_out, static_cast(value)); + return value_out + sizeof(std::int32_t); + } + if (value <= static_cast((std::numeric_limits::max)())) + { + out[0] = to_char_type(0x12); // int64 + store_little_endian(value_out, static_cast(value)); + return value_out + sizeof(std::int64_t); + } + out[0] = to_char_type(0x11); // uint64 + store_little_endian(value_out, value); + return value_out + sizeof(std::uint64_t); + } + + case value_t::null: + out[0] = to_char_type(0x0A); + return value_out; + + case value_t::string: + case value_t::binary: + case value_t::object: + case value_t::array: + case value_t::discarded: + default: + return nullptr; + } + } + + /*! + @brief Writes the BSON document entry with key @a name for @a j, which is + neither an object nor an array, with the key and a fixed-size value + (or the header of a string or binary value) in one reservation + @return whether the entry was written; false for a discarded value, a + string or binary value whose length does not fit the length + field, in which case nothing was written + @pre @a name is at most bson_scalar_max_name_size characters long and is + not changed by the error handler + */ + bool write_bson_scalar(const string_t& name, const BasicJsonType& j) + { + const std::size_t name_size = name.size(); + // a short string is written whole: its type, the name and its null + // byte, the length, the characters, and their null byte + const bool whole_string = j.is_string() && j.m_data.m_value.string->size() <= max_reserve_size - name_size - 7; + CharType* const out = oa.reserve(whole_string ? name_size + j.m_data.m_value.string->size() + 7 : name_size + 2 + 8); + // the name and its terminating null byte follow the type byte + CharType* p = out + 1; + copy_characters(p, reinterpret_cast(name.data()), name_size); + p += name_size; + *p++ = to_char_type(0x00); + + if (const CharType* const end = encode_bson_fixed_entry(out, p, j)) + { + oa.commit(static_cast(end - out)); + return true; + } + + if (whole_string) + { + const string_t& value = *j.m_data.m_value.string; + out[0] = to_char_type(0x02); + store_little_endian(p, static_cast(value.size() + 1ul)); + p += sizeof(std::int32_t); + copy_characters(p, reinterpret_cast(value.data()), value.size()); + p += value.size(); + // the terminating null byte is written explicitly rather than + // taken from the buffer, so that string_t::data() need not be + // null-terminated + *p++ = to_char_type(0x00); + oa.commit(static_cast(p - out)); + return true; + } + + if (j.is_string()) + { + const string_t& value = *j.m_data.m_value.string; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(value.size() + 1ul))) + { + return false; + } + out[0] = to_char_type(0x02); + store_little_endian(p, static_cast(value.size() + 1ul)); + p += sizeof(std::int32_t); + oa.commit(static_cast(p - out)); + // the terminating null byte is written explicitly rather than + // taken from the buffer, so that string_t::data() need not be + // null-terminated + oa.write_characters(reinterpret_cast(value.data()), value.size()); + oa.write_character(to_char_type(0x00)); + return true; + } + + if (j.is_binary()) + { + const binary_t& value = *j.m_data.m_value.binary; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(value.size()))) + { + return false; + } + out[0] = to_char_type(0x05); + store_little_endian(p, static_cast(value.size())); + p += sizeof(std::int32_t); + *p++ = value.has_subtype() ? to_char_type(static_cast(value.subtype())) : to_char_type(0x00); + oa.commit(static_cast(p - out)); + oa.write_characters(reinterpret_cast(value.data()), value.size()); + return true; + } + + return false; + } + /*! @brief Writes the BSON document entry with key @a name for @a j, which is neither an object nor an array */ void write_bson_value(const string_t& name, const BasicJsonType& j) { + // The strings are written as they are, as they are neither checked nor + // sanitized again here by these two error handlers. The name has to leave + // room for the larger payloads (see write_bson_scalar). + if (name.size() <= bson_scalar_max_name_size + && (error_handler == error_handler_t::keep || error_handler == error_handler_t::strict) + && write_bson_scalar(name, j)) + { + return; + } + switch (j.type()) { case value_t::binary: @@ -1757,6 +2052,9 @@ class binary_writer write_number(to_bson_length(size), true); } + /// space for the name of an array element, the decimal index, in the frame + static constexpr std::size_t bson_index_name_space = 24; + /// @brief an object or array of the BSON document being sized or written struct bson_frame { @@ -1770,28 +2068,196 @@ class binary_writer } } + /// @ref calc_bson_sizes only: make @ref index the next one and count the + /// digits of its name + void count_index() noexcept + { + if (++index == index_boundary) + { + ++index_name_size; + index_boundary *= 10; + } + } + + /// @ref write_bson_document only: make @ref index the next one and + /// update its decimal name + void next_index() noexcept + { + ++index; + std::size_t i = index_name_size; + while (i > 0 && index_name[i - 1] == '9') + { + index_name[--i] = '0'; + } + if (i > 0) + { + ++index_name[i - 1]; + } + else + { + // all digits were 9: 10, 100, ... + index_name[0] = '1'; + index_name[index_name_size++] = '0'; + } + } + /// the object or array const BasicJsonType* value; /// objects: the next member typename BasicJsonType::object_t::const_iterator member{}; /// arrays: the index of the next element std::size_t index = 0; + /// arrays: the name BSON gives that element, its decimal index (at most + /// 2^64 - 1, which has 20 digits), in space for copying a few characters + /// more; only maintained by @ref write_bson_document + std::array index_name{{'0'}}; + /// arrays: the number of digits of the index + std::size_t index_name_size = 1; + /// arrays: the first index with one more digit; only maintained by + /// @ref calc_bson_sizes + std::uint64_t index_boundary = 10; /// @ref calc_bson_sizes only: where its size goes in the table std::size_t size_slot; /// @ref calc_bson_sizes only: the size of its entries seen so far std::size_t entries_size = 0; }; - /*! - @brief creates the name BSON gives the array element with index @a index - @param[out] name receives the decimal index - */ - static void create_bson_index_name(const std::size_t index, string_t& name) + /// whether the BSON entry of @a j has a fixed size, which is the case for + /// a boolean, a number, and null + static bool is_bson_fixed_size(const BasicJsonType& j) noexcept { - // the index is built as a std::string; convert explicitly, as the - // two are only implicitly convertible for some string types - const auto key = std::to_string(index); - name = string_t(key.data(), key.size()); + return j.is_number() || j.is_boolean() || j.is_null(); + } + + /*! + @brief Writes the entries of the array in @a frame that have a fixed size + (booleans, numbers, and null), from its next element on, until it + reaches another element or the end of the array + + The index of the element and its name are advanced in @a frame. + + The entries are encoded straight into space reserved from the sink for as + many of them as fit, with the index, its digits, and the write pointer + kept in local variables: stores of characters may alias the frame, and + would force them to be loaded from it again for every element. + */ + void write_bson_array_fixed_entries(bson_frame& frame) + { + const auto& array = *frame.value->m_data.m_value.array; + const std::size_t count = array.size(); + std::size_t index = frame.index; + std::size_t name_size = frame.index_name_size; + std::array name = frame.index_name; + + // an entry is at most the type, 20 digits, the null byte, and 8 bytes + constexpr std::size_t max_entry = 1 + 20 + 1 + 8; + while (index < count) + { + CharType* const start = oa.reserve(max_reserve_size); + CharType* out = start; + const CharType* const limit = start + (max_reserve_size - max_entry); + while (out <= limit && index < count) + { + CharType* p = out + 1; + std::memcpy(p, name.data(), name.size()); + p += name_size; + *p++ = to_char_type(0x00); + CharType* const end = encode_bson_fixed_entry(out, p, array[index]); + if (end == nullptr) + { + break; + } + out = end; + + ++index; + std::size_t i = name_size; + while (i > 0 && name[i - 1] == '9') + { + name[--i] = '0'; + } + if (i > 0) + { + ++name[i - 1]; + } + else + { + // all digits were 9: 10, 100, ... + name[0] = '1'; + name[name_size++] = '0'; + } + } + oa.commit(static_cast(out - start)); + if (out <= limit && index < count) + { + // stopped at an element that has no fixed size + break; + } + } + + frame.index = index; + frame.index_name_size = name_size; + frame.index_name = name; + } + + /*! + @brief Calculates the size of the BSON array @a j if all its elements are + booleans, numbers, or null + @param[out] size the size of the array, including its own size field + @return false (and nothing is known about @a size) if an element is not, + or is an integer that does not fit 64 bits (which the generic + path rejects), which is not looked at further + */ + bool calc_bson_fixed_array_size(const BasicJsonType& j, std::size_t& size) const noexcept + { + std::size_t total = sizeof(std::int32_t) + 1ul; + std::size_t index_name_size = 1; + std::uint64_t index_boundary = 10; + std::uint64_t index = 0; + for (const auto& el : *j.m_data.m_value.array) + { + // the type, the decimal index, and the terminating null byte + total += 2ul + index_name_size; + if (++index == index_boundary) + { + ++index_name_size; + index_boundary *= 10; + } + switch (el.type()) + { + case value_t::boolean: + total += 1ul; + break; + case value_t::number_float: + total += 8ul; + break; + case value_t::number_integer: + // an integer beyond 64 bits is left to the generic path, which throws + if (JSON_HEDLEY_UNLIKELY(!integer_fits(el.m_data.m_value.number_integer))) + { + return false; + } + total += calc_bson_integer_size(static_cast(el.m_data.m_value.number_integer)); + break; + case value_t::number_unsigned: + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(el.m_data.m_value.number_unsigned))) + { + return false; + } + total += calc_bson_unsigned_size(conditional_static_cast(el.m_data.m_value.number_unsigned)); + break; + case value_t::null: + break; + case value_t::string: + case value_t::binary: + case value_t::object: + case value_t::array: + case value_t::discarded: + default: + return false; + } + } + size = total; + return true; } /*! @@ -1825,8 +2291,6 @@ class binary_writer // is in; nothing is allocated unless the document nests bson_frame current(&document); std::vector parents; - // string_t need not be default constructible - string_t index_name("", 0); while (true) { @@ -1857,9 +2321,10 @@ class binary_writer while (nested == nullptr && current.index < array.size()) { const BasicJsonType& el = array[current.index]; - create_bson_index_name(current.index, index_name); - current.entries_size += calc_bson_entry_header_size(index_name, el); - ++current.index; + // the type, the decimal index (which is valid UTF-8 and + // has no U+0000), and the terminating null byte + current.entries_size += 2ul + current.index_name_size; + current.count_index(); if (el.is_structured()) { nested = ⪙ @@ -1873,6 +2338,15 @@ class binary_writer if (nested != nullptr) { + // an array of booleans, numbers and null: sized without a frame + std::size_t fixed_size = 0; + if (nested->is_array() && calc_bson_fixed_array_size(*nested, fixed_size)) + { + nested_sizes.push_back(fixed_size); + current.entries_size += fixed_size; + continue; + } + // its size is added to the current one's once it is done nested_sizes.push_back(0); parents.push_back(std::move(current)); @@ -1914,8 +2388,6 @@ class binary_writer bson_frame current(&document); std::vector parents; std::size_t next_size = 0; - // string_t need not be default constructible - string_t index_name("", 0); while (true) { @@ -1945,9 +2417,19 @@ class binary_writer const auto& array = *current.value->m_data.m_value.array; while (nested == nullptr && current.index < array.size()) { + if (is_bson_fixed_size(array[current.index])) + { + write_bson_array_fixed_entries(current); + if (current.index == array.size()) + { + break; + } + } + + // a string, binary value, object or array const BasicJsonType& el = array[current.index]; - create_bson_index_name(current.index, index_name); - ++current.index; + const string_t index_name(current.index_name.data(), current.index_name_size); + current.next_index(); if (el.is_structured()) { write_bson_nested_head(index_name, el, nested_sizes[next_size++]); @@ -1962,6 +2444,20 @@ class binary_writer if (nested != nullptr) { + if (nested->is_array()) + { + // an array of booleans, numbers and null is written without a frame on the stack + bson_frame leaf(nested); + write_bson_array_fixed_entries(leaf); + if (leaf.index == nested->m_data.m_value.array->size()) + { + oa.write_character(to_char_type(0x00)); + continue; + } + parents.push_back(std::move(current)); + current = std::move(leaf); + continue; + } parents.push_back(std::move(current)); current = bson_frame(nested); continue; @@ -2124,6 +2620,68 @@ class binary_writer return encode_first_byte_and_tail(out, first_byte, size_class, argument); } + /// write the head of an array or object: one byte for a size of up to 23, else as + /// @ref write_cbor_head + void write_cbor_container_head(const std::uint8_t major_type, const std::uint64_t size) + { + if (JSON_HEDLEY_LIKELY(size <= 0x17)) + { + oa.write_character(to_char_type(static_cast(major_type + size))); + return; + } + write_cbor_head(major_type, size); + } + + /*! + @brief write an object value (head and members) as CBOR + + A member whose value is null, a boolean, an integer, or a string is written + without a call of @ref write_cbor. Kept out of line, like @ref write_cbor_array. + */ + JSON_HEDLEY_NEVER_INLINE + void write_cbor_object(const BasicJsonType& j, const std::size_t depth) + { + // step 1: write control byte and the object size + write_cbor_container_head(0xA0, j.m_data.m_value.object->size()); + + for (const auto& el : *j.m_data.m_value.object) + { + // el.first is written directly (not via a temporary + // basic_json), with the object as diagnostics context + write_cbor_key(el.first, j); + + const BasicJsonType& value = el.second; + switch (value.type()) + { + case value_t::null: + oa.write_character(to_char_type(0xF6)); + break; + + case value_t::boolean: + oa.write_character(value.m_data.m_value.boolean ? to_char_type(0xF5) : to_char_type(0xF4)); + break; + + case value_t::number_integer: + case value_t::number_unsigned: + write_cbor_integer(value, std::integral_constant()); + break; + + case value_t::string: + write_cbor_string(*value.m_data.m_value.string, value); + break; + + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::number_float: + case value_t::discarded: + default: + write_cbor(value, depth + 1); + break; + } + } + } + /*! @brief write an array value (head and elements) as CBOR @@ -2135,15 +2693,22 @@ class binary_writer JSON_HEDLEY_NEVER_INLINE void write_cbor_array(const BasicJsonType& j, const std::size_t depth) { - write_cbor_head(0x80, j.m_data.m_value.array->size()); - write_cbor_array_elements(j, depth, std::integral_constant()); + write_cbor_container_head(0x80, j.m_data.m_value.array->size()); + if (!j.m_data.m_value.array->empty()) + { + write_cbor_array_elements(j, depth, std::integral_constant()); + } } + JSON_HEDLEY_NEVER_INLINE void write_cbor_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/) { write_array_elements(j, [this](CharType * out, const BasicJsonType & el) { return encode_cbor_integer(out, el); + }, [this](CharType * out, const number_float_t n) + { + return encode_cbor_float(out, n); }, [this, depth](const BasicJsonType & el) { write_cbor(el, depth + 1); @@ -2160,22 +2725,28 @@ class binary_writer /*! @brief write the elements of the array @a j, runs of integers with - @a encode_integer and all other values with @a write_value + @a encode_integer, runs of floats with @a encode_float, and all + other values with @a write_value - Runs of integers are encoded straight into space reserved from the sink + Runs of numbers are encoded straight into space reserved from the sink for a whole run: the write pointer stays a local, where it would otherwise be reloaded from the sink for every value, as a store of a character may alias it. */ - template - void write_array_elements(const BasicJsonType& j, EncodeInteger encode_integer, WriteValue write_value) + template + void write_array_elements(const BasicJsonType& j, EncodeInteger encode_integer, EncodeFloat encode_float, WriteValue write_value) { constexpr std::size_t max_run = max_reserve_size / integer_encoding_space; auto it = j.m_data.m_value.array->begin(); const auto end = j.m_data.m_value.array->end(); while (it != end) { - if (it->is_number_integer()) + if (!it->is_number()) + { + write_value(*it); + ++it; + } + else if (!it->is_number_float()) { CharType* const start = oa.reserve(max_run * integer_encoding_space); CharType* out = start; @@ -2187,12 +2758,33 @@ class binary_writer } else { - write_value(*it); - ++it; + it = write_float_elements(it, end, encode_float); } } } + /*! + @brief write the run of floats that starts at @a it with @a encode_float + @return the first element after the run + + Out of line, so that the loop of integers in @ref write_array_elements is + not changed by it. + */ + template + JSON_HEDLEY_NEVER_INLINE + Iterator write_float_elements(Iterator it, const Iterator end, EncodeFloat encode_float) // NOLINT(performance-unnecessary-value-param): iterators are passed by value + { + constexpr std::size_t max_run = max_reserve_size / integer_encoding_space; + CharType* const start = oa.reserve(max_run * integer_encoding_space); + CharType* out = start; + for (std::size_t n = 0; n < max_run && it != end && it->is_number_float(); ++n, ++it) + { + out += encode_float(out, it->m_data.m_value.number_float); + } + oa.commit(static_cast(out - start)); + return it; + } + /*! @brief write a number_integer or number_unsigned value as a CBOR integer @@ -2368,6 +2960,64 @@ class binary_writer } } + /// write a float as MessagePack, into space reserved from the sink + JSON_HEDLEY_NEVER_INLINE + void write_msgpack_float(const number_float_t n) + { + oa.commit(encode_compact_float(oa.reserve(integer_encoding_space), n, to_char_type(0xCA), to_char_type(0xCB))); + } + + /*! + @brief write an object value (head and members) as MessagePack + + See @ref write_cbor_object. + */ + JSON_HEDLEY_NEVER_INLINE + void write_msgpack_object(const BasicJsonType& j, const std::size_t depth) + { + write_msgpack_object_prefix(j.m_data.m_value.object->size(), j); + + for (const auto& el : *j.m_data.m_value.object) + { + // as in write_cbor, el.first is written directly with the + // object as diagnostics context + write_msgpack_key(el.first, j); + + const BasicJsonType& value = el.second; + switch (value.type()) + { + case value_t::null: + oa.write_character(to_char_type(0xC0)); + break; + + case value_t::boolean: + oa.write_character(value.m_data.m_value.boolean ? to_char_type(0xC3) : to_char_type(0xC2)); + break; + + case value_t::number_integer: + case value_t::number_unsigned: + write_msgpack_integer(value, std::integral_constant()); + break; + + case value_t::number_float: + write_msgpack_float(value.m_data.m_value.number_float); + break; + + case value_t::string: + write_msgpack_string(*value.m_data.m_value.string, value); + break; + + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + write_msgpack(value, depth + 1); + break; + } + } + } + /*! @brief write an array value (head and elements) as MessagePack @@ -2377,14 +3027,21 @@ class binary_writer void write_msgpack_array(const BasicJsonType& j, const std::size_t depth) { write_msgpack_array_prefix(j.m_data.m_value.array->size(), j); - write_msgpack_array_elements(j, depth, std::integral_constant()); + if (!j.m_data.m_value.array->empty()) + { + write_msgpack_array_elements(j, depth, std::integral_constant()); + } } + JSON_HEDLEY_NEVER_INLINE void write_msgpack_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/) { write_array_elements(j, [this](CharType * out, const BasicJsonType & el) { return encode_msgpack_integer(out, el); + }, [this](CharType * out, const number_float_t n) + { + return encode_compact_float(out, n, to_char_type(0xCA), to_char_type(0xCB)); }, [this, depth](const BasicJsonType & el) { write_msgpack(el, depth + 1); @@ -2436,6 +3093,24 @@ class binary_writer write_msgpack(BasicJsonType(key)); } + /// the longest string that is written, with its head of one or two + /// characters, in a single reservation from the sink + static constexpr std::size_t max_text_in_reservation = max_reserve_size - 2; + + /// copy the characters of a string to space in the sink; a short one + /// without a call of memcpy() + static void copy_characters(CharType* dst, const CharType* src, const std::size_t length) noexcept + { + if (length <= 32) + { + copy_short(dst, src, length, std::integral_constant()); + } + else + { + std::memcpy(dst, src, length * sizeof(CharType)); + } + } + /*! @brief write a CBOR text string @@ -2451,6 +3126,37 @@ class binary_writer this function. */ void write_cbor_string(const string_t& value, const BasicJsonType& context) + { + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + // the string is written as it is: head and characters in one reservation + if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 0x17) + { + *p++ = to_char_type(static_cast(0x60 + value.size())); + } + else + { + *p++ = to_char_type(0x78); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + return; + } + } + + write_cbor_checked_string(value, context); + } + + /// @brief write a CBOR text string checked or sanitized by the error handler; + /// out of line, as the string_t for the sanitized copy would make + /// every call of @ref write_cbor_string set up a frame for it + JSON_HEDLEY_NEVER_INLINE + void write_cbor_checked_string(const string_t& value, const BasicJsonType& context) { string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -2479,6 +3185,38 @@ class binary_writer @ref sanitize_utf8_for_write must not escape this function. */ void write_msgpack_string(const string_t& value, const BasicJsonType& context) + { + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + // the string is written as it is: head and characters in one reservation + if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 31) + { + // fixstr + *p++ = to_char_type(static_cast(0xA0 | value.size())); + } + else + { + // str 8 + *p++ = to_char_type(0xD9); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + return; + } + } + + write_msgpack_checked_string(value, context); + } + + /// @brief write a MessagePack str checked or sanitized by the error handler; + /// out of line, like @ref write_cbor_checked_string + JSON_HEDLEY_NEVER_INLINE + void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context) { string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -2548,6 +3286,375 @@ class binary_writer write_ubjson_integer_payload(prefix, n, use_bjdata); } + /*! + @brief markers and size classes of UBJSON/BJData integers, by the bit length + of the magnitude + + A non-negative integer has the magnitude n, a negative one ~n = -1 - n; the + bit length of the magnitude is 1 to 64. The first 130 characters of the + table are the markers: 65 for a non-negative integer, then 65 for a negative + one (index + 65). The marker is the first of `i`, `U`, `I`, `u` (BJData), + `l`, `m` (BJData), `L`, `M` (BJData) whose range contains the integer; `H` + (a high-precision number) is for a non-negative integer of 64 bits in + UBJSON. The following 130 characters are the size classes at index + 130: + the digit 0, 1, 2, or 3 for 1, 2, 4, or 8 bytes. Looked up, like + @ref integer_size_class, instead of found by branches or comparisons. + + @sa ubjson_integer_prefix for the integers of types of more than 64 bits + */ + static const char* ubjson_integer_table(const bool use_bjdata) noexcept + { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): initialized from string literals + static constexpr char ubjson_table[] = + // the markers for n >= 0, then n < 0 + "i" "iiiiiii" "U" "IIIIIII" "llllllllllllllll" "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL" "H" + "i" "iiiiiii" "IIIIIIII" "llllllllllllllll" "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL" "L" + // the size classes + "000000000" "1111111" "2222222222222222" "333333333333333333333333333333333" + "00000000" "11111111" "2222222222222222" "333333333333333333333333333333333"; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): initialized from string literals + static constexpr char bjdata_table[] = + "i" "iiiiiii" "U" "IIIIIII" "u" "lllllllllllllll" "m" "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL" "M" + "i" "iiiiiii" "IIIIIIII" "llllllllllllllll" "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL" "L" + "000000000" "11111111" "2222222222222222" "33333333333333333333333333333333" + "00000000" "11111111" "2222222222222222" "333333333333333333333333333333333"; + static_assert(sizeof(ubjson_table) == 261 && sizeof(bjdata_table) == 261, "two tables of 130 markers and 130 size classes"); + return use_bjdata ? bjdata_table : ubjson_table; + } + + /// the index in ubjson_integer_table() of the integer with the 64 bits + /// @a raw and the sign mask @a sign_mask (all ones if it is negative) + static std::size_t ubjson_integer_index(const std::uint64_t raw, const std::uint64_t sign_mask) noexcept + { + const std::size_t sign_offset = sign_mask != 0 ? std::size_t{65} : + std::size_t{0}; + return static_cast(64 - count_leading_zeros((raw ^ sign_mask) | 1u)) + sign_offset; + } + + /*! + @brief encode a UBJSON/BJData integer without branching on its size + @param[in] out where the encoding goes; at most 9 characters are written + @param[in] raw the 64 bits of the integer (two's complement) + @param[in] sign_mask all ones if the integer is negative, else zero + @param[in] table ubjson_integer_table(use_bjdata) + @return the end of the encoding, or nullptr if the integer is a high-precision + number (a non-negative one of 64 bits in UBJSON), which is not written + */ + CharType* encode_ubjson_integer(CharType* out, const std::uint64_t raw, const std::uint64_t sign_mask, + const bool add_prefix, const bool use_bjdata, const char* table) const noexcept + { + const std::size_t index = ubjson_integer_index(raw, sign_mask); + const char marker = table[index]; + if (JSON_HEDLEY_UNLIKELY(marker == 'H')) + { + return nullptr; + } + const unsigned size_class = static_cast(table[index + 130]) & 3u; + + // the marker (or its place, if not written) and one 8-byte word, whose + // bytes after the integer are scratch that the caller drops + out[0] = to_char_type(static_cast(marker)); + out += add_prefix ? 1 : 0; + const unsigned unused_bits = 64u - (8u << size_class); + // UBJSON in network order, BJData in little-endian + std::uint64_t tail = 0; + if (use_bjdata) + { + tail = is_little_endian ? raw : byteswap(raw); + } + else + { + tail = is_little_endian ? byteswap(raw) >> unused_bits : raw << unused_bits; + } + std::memcpy(out, &tail, sizeof(tail)); + return out + (std::size_t{1} << size_class); + } + + /// @brief whether the integers of BasicJsonType are written by + /// @ref encode_ubjson_integer + static constexpr bool ubjson_integers_fit_64_bits() noexcept + { + return integers_fit_64_bits(); + } + + /// space for the largest encoding of encode_ubjson_integer(): the marker and 8 bytes + static constexpr std::size_t ubjson_integer_encoding_space = 9; + + /*! + @brief write the bytes of a binary value as an array of UBJSON uint8 values + + Every byte is written with its marker, in blocks that fit the space reserved + from the sink. Kept out of line, so that write_ubjson does not carry the + registers of its loop. + */ + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_binary_bytes(const binary_t& binary, const bool bjdata_draft3) + { + const CharType marker = to_char_type(bjdata_draft3 ? 'B' : 'U'); + const auto* const data = binary.data(); + const std::size_t size = binary.size(); + for (std::size_t i = 0; i < size;) + { + const std::size_t block = (std::min)(size - i, max_reserve_size / 2); + CharType* const out = oa.reserve(2 * block); + for (std::size_t k = 0; k < block; ++k) + { + out[2 * k] = marker; + // the cast is needed for binary types whose value type + // is not an integer (e.g., std::byte) + out[(2 * k) + 1] = to_char_type(static_cast(data[i + k])); + } + i += block; + oa.commit(2 * block); + } + } + + /*! + @brief write the members of the object @a j of a UBJSON/BJData container + + A member whose value is null, a boolean, an integer, a float, or a string is + written without a call of @ref write_ubjson. Kept out of line, like + @ref write_cbor_object. + */ + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_members(const BasicJsonType& j, const bool use_count, const bool use_type, + const bool prefix_required, const bool use_bjdata, const bjdata_version_t bjdata_version, + const std::size_t depth) + { + for (const auto& el : *j.m_data.m_value.object) + { + // a converted key must outlive the reference returned by sanitize_utf8_for_write + const object_key_string_t key_string = el.first; + write_ubjson_text(key_string, j, false, use_bjdata); + + const BasicJsonType& value = el.second; + switch (value.type()) + { + case value_t::null: + if (prefix_required) + { + oa.write_character(to_char_type('Z')); + } + break; + + case value_t::boolean: + if (prefix_required) + { + oa.write_character(value.m_data.m_value.boolean ? to_char_type('T') : to_char_type('F')); + } + break; + + case value_t::number_integer: + write_number_with_ubjson_prefix(value.m_data.m_value.number_integer, prefix_required, use_bjdata); + break; + + case value_t::number_unsigned: + write_number_with_ubjson_prefix(value.m_data.m_value.number_unsigned, prefix_required, use_bjdata); + break; + + case value_t::number_float: + write_number_with_ubjson_prefix(value.m_data.m_value.number_float, prefix_required, use_bjdata); + break; + + case value_t::string: + write_ubjson_text(*value.m_data.m_value.string, value, prefix_required, use_bjdata); + break; + + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + write_ubjson(value, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + break; + } + } + } + + /// write the count marker `#` and the number of elements of a container as + /// an integer; a count of up to 127, which is `i` and a byte, in one reservation + void write_ubjson_count(const std::size_t count, const bool use_bjdata) + { + if (JSON_HEDLEY_LIKELY(count <= 127)) + { + CharType* const out = oa.reserve(3); + out[0] = to_char_type('#'); + out[1] = to_char_type('i'); + out[2] = to_char_type(static_cast(count)); + oa.commit(3); + return; + } + oa.write_character(to_char_type('#')); + write_number_with_ubjson_prefix(count, true, use_bjdata); + } + + /*! + @brief write a UBJSON/BJData string or object key: the optional string + marker `S`, the length as an integer with its marker, and the characters + + With the default error handler (keep) and a string of up to 253 characters, + which has a length of the marker `i` or `U` and a single byte, everything is + written into space reserved from the sink in one go. + + @param[in] value the string; checked or sanitized according to + @ref error_handler, with @a context as diagnostics context + @param[in] add_marker whether `S` is written first (not for an object key) + */ + void write_ubjson_text(const string_t& value, const BasicJsonType& context, const bool add_marker, const bool use_bjdata) + { + const std::size_t size = value.size(); + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep) && JSON_HEDLEY_LIKELY(size <= max_text_in_reservation - 1)) + { + CharType* const out = oa.reserve(size + 3); + CharType* p = out; + if (add_marker) + { + *p++ = to_char_type('S'); + } + *p++ = to_char_type(size <= 127 ? 'i' : 'U'); + *p++ = to_char_type(static_cast(size)); + copy_characters(p, reinterpret_cast(value.data()), size); + oa.commit(static_cast(p - out) + size); + return; + } + + write_ubjson_checked_text(value, context, add_marker, use_bjdata); + } + + /// @brief write a UBJSON/BJData string checked or sanitized by the error + /// handler; out of line, like @ref write_cbor_checked_string + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_checked_text(const string_t& value, const BasicJsonType& context, const bool add_marker, const bool use_bjdata) + { + string_t storage; + const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); + if (add_marker) + { + oa.write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(sanitized.size(), true, use_bjdata); + oa.write_characters( + reinterpret_cast(sanitized.data()), + sanitized.size()); + } + + /*! + @brief encode a UBJSON/BJData float, with its marker if @a add_prefix + @param[in] out where the encoding goes; at most 9 characters are written + @return the end of the encoding + */ + CharType* encode_ubjson_float(CharType* out, const number_float_t n, const bool add_prefix, const bool use_bjdata) const noexcept + { + if (add_prefix) + { + *out++ = get_ubjson_float_prefix(); + } + return store_number(out, n, use_bjdata); + } + + /// whether @a array starts with a number that @ref write_ubjson_array_elements + /// can encode (all but an integer that UBJSON writes as a high-precision + /// number); an array that does not is written by write_ubjson one element at a time + template + static bool starts_with_ubjson_number(const ArrayType& array, const bool use_bjdata, std::true_type /*fits_64_bits*/) noexcept + { + if (array.empty() || !array.front().is_number()) + { + return false; + } + const BasicJsonType& first = array.front(); + return use_bjdata || first.m_data.m_type != value_t::number_unsigned + || (static_cast(first.m_data.m_value.number_unsigned) >> 63u) == 0; + } + + template + static bool starts_with_ubjson_number(const ArrayType& /*array*/, const bool /*use_bjdata*/, std::false_type /*fits_64_bits*/) noexcept + { + return false; + } + + /*! + @brief write the elements of an array of a UBJSON/BJData container that + starts with a number, with runs of numbers encoded without a call of + @ref write_ubjson for each + + Same output as calling write_ubjson for each element. The numbers of a + run are encoded straight into space reserved from the sink, as in + @ref write_array_elements; all other values, and an integer that UBJSON + writes as a high-precision number, are written with write_ubjson. Kept out + of line, like @ref write_cbor_array. + */ + template + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_array_elements(const ArrayType& array, const bool use_count, const bool use_type, + const bool prefix_required, const bool use_bjdata, const bjdata_version_t bjdata_version, + const std::size_t depth) + { + constexpr std::size_t max_run = max_reserve_size / ubjson_integer_encoding_space; + const char* const table = ubjson_integer_table(use_bjdata); + auto it = array.begin(); + const auto end = array.end(); + while (it != end) + { + if (!it->is_number()) + { + write_ubjson(*it, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + ++it; + continue; + } + + CharType* const start = oa.reserve(max_run * ubjson_integer_encoding_space); + CharType* out = start; + bool unencodable = false; + for (std::size_t n = 0; n < max_run && it != end && it->is_number(); ++n) + { + CharType* next = nullptr; + if (it->is_number_float()) + { + next = encode_ubjson_float(out, it->m_data.m_value.number_float, prefix_required, use_bjdata); + } + else + { + std::uint64_t sign_mask = 0; + const std::uint64_t raw = integer_bits(*it, sign_mask); + next = encode_ubjson_integer(out, raw, sign_mask, prefix_required, use_bjdata, table); + } + if (JSON_HEDLEY_UNLIKELY(next == nullptr)) + { + unencodable = true; + break; + } + out = next; + ++it; + } + oa.commit(static_cast(out - start)); + + if (unencodable) + { + write_ubjson(*it, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + ++it; + } + } + } + + /*! + @brief store @a n at @a out in the byte order of the output + @param[in] output_little_endian whether the output is little-endian + @return the end of the number + */ + template + CharType* store_number(CharType* out, const NumberType n, const bool output_little_endian) const noexcept + { + std::array vec{}; + std::memcpy(vec.data(), &n, sizeof(NumberType)); + if (is_little_endian != output_little_endian) + { + reverse_bytes(vec); + } + std::memcpy(out, vec.data(), sizeof(NumberType)); + return out + sizeof(NumberType); + } + /*! @brief determine the UBJSON/BJData type marker of an integer @@ -2638,19 +3745,58 @@ class binary_writer break; default: { - // high-precision number: the decimal digits as a string JSON_ASSERT(prefix == 'H'); - const auto number = BasicJsonType(n).dump(); - write_number_with_ubjson_prefix(number.size(), true, use_bjdata); - for (std::size_t i = 0; i < number.size(); ++i) - { - oa.write_character(to_char_type(static_cast(number[i]))); - } + write_ubjson_high_precision(n, use_bjdata); break; } } } + /// @brief write an integer as a high-precision number: its decimal digits as + /// a string; out of line, as the string would otherwise burden + /// @ref write_ubjson_integer_payload for every integer + template + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_high_precision(const NumberType n, const bool use_bjdata) + { + write_ubjson_high_precision(n, use_bjdata, std::integral_constant < bool, std::is_unsigned::value && sizeof(NumberType) <= sizeof(std::uint64_t) > ()); + } + + /// an unsigned integer of at most 64 bits: its decimal digits are written directly + template + void write_ubjson_high_precision(const NumberType n, const bool use_bjdata, std::true_type /*is_unsigned_64_bits*/) + { + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::size_t first = digits.size(); + auto value = static_cast(n); + do + { + digits[--first] = static_cast('0' + (value % 10)); + value /= 10; + } + while (value != 0); + const std::size_t length = digits.size() - first; + write_number_with_ubjson_prefix(length, true, use_bjdata); + CharType* const out = oa.reserve(digits.size()); + for (std::size_t i = 0; i < length; ++i) + { + out[i] = to_char_type(static_cast(digits[first + i])); + } + oa.commit(length); + } + + /// any other integer: its decimal digits from basic_json::dump() + template + void write_ubjson_high_precision(const NumberType n, const bool use_bjdata, std::false_type /*is_unsigned_64_bits*/) + { + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true, use_bjdata); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa.write_character(to_char_type(static_cast(number[i]))); + } + } + /*! @brief determine the type prefix of container values */ @@ -2728,8 +3874,8 @@ class binary_writer static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) { return el.is_number_unsigned() - ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(el.template get()) - : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(el.template get()); + ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(static_cast(el.m_data.m_value.number_unsigned)) + : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(static_cast(el.m_data.m_value.number_integer)); } /*! @@ -2818,7 +3964,7 @@ class binary_writer */ bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run) { - const auto dval = el.template get(); + const auto dval = static_cast(el.m_data.m_value.number_float); if (dry_run) { #ifdef __GNUC__ @@ -2842,6 +3988,90 @@ class binary_writer return true; } + /*! + @brief write the elements of an ND-array as the number kind @a dtype names + (see @ref write_bjdata_ndarray), after they have been found to be + in range by a dry run of @ref write_bjdata_ndarray_elements + + The elements are encoded in blocks into space reserved from the sink. + */ + void write_bjdata_ndarray_values(const BasicJsonType& array_data, const CharType dtype) + { + switch (dtype) + { + case 'U': + case 'C': + case 'B': + write_bjdata_ndarray_block(array_data); + break; + case 'i': + write_bjdata_ndarray_block(array_data); + break; + case 'u': + write_bjdata_ndarray_block(array_data); + break; + case 'I': + write_bjdata_ndarray_block(array_data); + break; + case 'm': + write_bjdata_ndarray_block(array_data); + break; + case 'l': + write_bjdata_ndarray_block(array_data); + break; + case 'M': + write_bjdata_ndarray_block(array_data); + break; + case 'L': + write_bjdata_ndarray_block(array_data); + break; + case 'd': + write_bjdata_ndarray_block(array_data); + break; + case 'D': + default: + // 'D' (double) already spans the full range of number_float_t + write_bjdata_ndarray_block(array_data); + break; + } + } + + /// the value of an ND-array element that is an integer, as the integer type that @a T is converted from + template + static T bjdata_ndarray_integer(const BasicJsonType& el, std::true_type /*is_integral*/) noexcept + { + using storage_type = typename std::conditional::value, std::uint64_t, std::int64_t>::type; + return static_cast(el.is_number_unsigned() + ? static_cast(el.m_data.m_value.number_unsigned) + : static_cast(el.m_data.m_value.number_integer)); + } + + /// the value of an ND-array element that is a float + template + static T bjdata_ndarray_integer(const BasicJsonType& el, std::false_type /*is_integral*/) noexcept + { + return static_cast(static_cast(el.m_data.m_value.number_float)); + } + + /// write the elements of an ND-array as @a T, little-endian, in blocks + template + void write_bjdata_ndarray_block(const BasicJsonType& array_data) + { + constexpr std::size_t per_block = max_reserve_size / sizeof(T); + auto it = array_data.m_data.m_value.array->begin(); + const auto end = array_data.m_data.m_value.array->end(); + while (it != end) + { + CharType* const start = oa.reserve(per_block * sizeof(T)); + CharType* out = start; + for (std::size_t n = 0; n < per_block && it != end; ++n, ++it) + { + out = store_number(out, bjdata_ndarray_integer(*it, std::is_integral()), true); + } + oa.commit(static_cast(out - start)); + } + } + /*! @brief validate or write every element of a BJData ND-array's `_ArrayData_` @param[in] array_data the `_ArrayData_` array @@ -2851,6 +4081,12 @@ class binary_writer */ bool write_bjdata_ndarray_elements(const BasicJsonType& array_data, const CharType dtype, const bool dry_run) { + if (!dry_run) + { + write_bjdata_ndarray_values(array_data, dtype); + return true; + } + for (const auto& el : array_data) { bool ok = true; @@ -3136,9 +4372,16 @@ class binary_writer // 0x80..0x84: array with 0..4 elements; 0x85: array ended by 0xFE write_bon8_marker(static_cast(N <= 4 ? 0x80 + N : 0x85), string_open); - for (const auto& el : *j.m_data.m_value.array) + if (N != 0 && j.m_data.m_value.array->front().is_number()) { - write_bon8_value(el, string_open, depth + 1); + write_bon8_array_elements(*j.m_data.m_value.array, string_open, depth); + } + else + { + for (const auto& el : *j.m_data.m_value.array) + { + write_bon8_value(el, string_open, depth + 1); + } } if (N > 4) @@ -3154,11 +4397,7 @@ class binary_writer // 0x86..0x8A: object with 0..4 members; 0x8B: object ended by 0xFE write_bon8_marker(static_cast(N <= 4 ? 0x86 + N : 0x8B), string_open); - for (const auto& el : *j.m_data.m_value.object) - { - write_bon8_string(el.first, string_open, j); - write_bon8_value(el.second, string_open, depth + 1); - } + write_bon8_members(j, string_open, depth); if (N > 4) { @@ -3169,22 +4408,7 @@ class binary_writer case value_t::binary: { - // BON8 has no binary type: write the bytes as an array of - // integers, like UBJSON and BJData do - const auto N = j.m_data.m_value.binary->size(); - write_bon8_marker(static_cast(N <= 4 ? 0x80 + N : 0x85), string_open); - - for (std::size_t i = 0; i < N; ++i) - { - // the cast is needed for binary types whose value type - // is not an integer (e.g., std::byte) - write_bon8_integer(static_cast(j.m_data.m_value.binary->data()[i])); - } - - if (N > 4) - { - write_bon8_marker(0xFE, string_open); - } + write_bon8_binary(*j.m_data.m_value.binary, string_open); break; } @@ -3271,6 +4495,281 @@ class binary_writer } } + /*! + @brief encode an integer as @ref write_bon8_integer writes it + @param[in] out where the encoding goes; at most 9 characters are written + @return the end of the encoding + */ + CharType* encode_bon8_integer(CharType* out, std::int64_t value) const noexcept + { + if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) + { + *out++ = to_char_type(0x8D); + return store_number(out, value, false); + } + if (value < -33818506 || value > 67637031) + { + *out++ = to_char_type(0x8C); + return store_number(out, static_cast(value), false); + } + if (value <= -264075) + { + value = -(value + 264075); + return put_bon8_bytes(out, 0xF0 + ((value >> 22) & 0x07), 0xC0 + ((value >> 16) & 0x3F), value >> 8, value); + } + if (value <= -1931) + { + value = -(value + 1931); + return put_bon8_bytes(out, 0xE0 + ((value >> 14) & 0x0F), 0xC0 + ((value >> 8) & 0x3F), value); + } + if (value <= -11) + { + value = -(value + 11); + return put_bon8_bytes(out, 0xC2 + ((value >> 6) & 0x1F), 0xC0 + (value & 0x3F)); + } + if (value <= -1) + { + return put_bon8_bytes(out, 0xB8 - (value + 1)); + } + if (value <= 39) + { + return put_bon8_bytes(out, 0x90 + value); + } + if (value <= 3879) + { + value -= 40; + return put_bon8_bytes(out, 0xC2 + ((value >> 7) & 0x1F), value & 0x7F); + } + if (value <= 528167) + { + value -= 3880; + return put_bon8_bytes(out, 0xE0 + ((value >> 15) & 0x0F), (value >> 8) & 0x7F, value); + } + value -= 528168; + return put_bon8_bytes(out, 0xF0 + ((value >> 23) & 0x07), (value >> 16) & 0x7F, value >> 8, value); + } + + /// store the low byte of each argument at @a out; return the end + template + CharType* put_bon8_bytes(CharType* out, const Bytes... bytes) const noexcept + { + const std::array buffer{{to_char_type(static_cast(bytes & 0xFF))...}}; + std::memcpy(out, buffer.data(), buffer.size()); + return out + buffer.size(); + } + + /*! + @brief encode a float as @ref write_bon8_float writes it + @param[in] out where the encoding goes; at most 9 characters are written + @return the end of the encoding + */ + CharType* encode_bon8_float(CharType* out, const number_float_t n) const noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + if (n == static_cast(-1)) + { + *out++ = to_char_type(0xFB); + return out; + } + if (n == static_cast(0) && !std::signbit(n)) + { + *out++ = to_char_type(0xFC); + return out; + } + if (n == static_cast(1)) + { + *out++ = to_char_type(0xFD); + return out; + } + if (std::isnan(n)) + { + return put_bon8_bytes(out, 0x8E, 0x7F, 0x80, 0x00, 0x01); + } + return out + encode_compact_float(out, n, to_char_type(0x8E), to_char_type(0x8F)); +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + } + + /*! + @brief write the elements of an array that starts with a number, with runs + of numbers encoded without a call of @ref write_bon8_value for each + + Same output as calling write_bon8_value for each element. The numbers of a + run are encoded into space reserved from the sink, as in + @ref write_array_elements; an integer that BON8 cannot write (an unsigned + one above the int64 range, or one of a number type wider than 64 bits) throws + from write_bon8_value, like any other value that is not a number. Kept out + of line, like @ref write_cbor_array. + */ + template + JSON_HEDLEY_NEVER_INLINE + void write_bon8_array_elements(const ArrayType& array, bool& string_open, const std::size_t depth) + { + constexpr std::size_t max_run = max_reserve_size / 9; + auto it = array.begin(); + const auto end = array.end(); + while (it != end) + { + if (!it->is_number()) + { + write_bon8_value(*it, string_open, depth + 1); + ++it; + continue; + } + + CharType* const start = oa.reserve(max_run * 9); + CharType* out = start; + bool unencodable = false; + for (std::size_t n = 0; n < max_run && it != end && it->is_number(); ++n) + { + if (it->is_number_float()) + { + out = encode_bon8_float(out, it->m_data.m_value.number_float); + } + else if (JSON_HEDLEY_UNLIKELY((it->m_data.m_type == value_t::number_unsigned + && !integer_fits_max(it->m_data.m_value.number_unsigned)) + || (it->m_data.m_type == value_t::number_integer + && !integer_fits(it->m_data.m_value.number_integer)))) + { + unencodable = true; + break; + } + else + { + out = encode_bon8_integer(out, it->m_data.m_type == value_t::number_integer + ? static_cast(it->m_data.m_value.number_integer) + : static_cast(it->m_data.m_value.number_unsigned)); + } + ++it; + } + oa.commit(static_cast(out - start)); + if (out != start) + { + // a number follows a string without terminating it + string_open = false; + } + + if (unencodable) + { + write_bon8_value(*it, string_open, depth + 1); + ++it; + } + } + } + + /*! + @brief write the members of the object @a j as BON8 + + A member whose value is null, a boolean, a number, or a string is written + without a call of @ref write_bon8_value. Kept out of line, like + @ref write_cbor_object. + */ + JSON_HEDLEY_NEVER_INLINE + void write_bon8_members(const BasicJsonType& j, bool& string_open, const std::size_t depth) + { + for (const auto& el : *j.m_data.m_value.object) + { + write_bon8_string(el.first, string_open, j); + + const BasicJsonType& value = el.second; + switch (value.type()) + { + case value_t::null: + write_bon8_marker(0xFA, string_open); + break; + + case value_t::boolean: + write_bon8_marker(value.m_data.m_value.boolean ? 0xF9 : 0xF8, string_open); + break; + + case value_t::string: + write_bon8_string(*value.m_data.m_value.string, string_open, value); + break; + + case value_t::number_integer: + if (JSON_HEDLEY_LIKELY(integer_fits(value.m_data.m_value.number_integer))) + { + write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); + string_open = false; + break; + } + // beyond 64 bits (number types wider than that): throws + write_bon8_value(value, string_open, depth + 1); + break; + + case value_t::number_float: + write_bon8_float(value.m_data.m_value.number_float); + string_open = false; + break; + + case value_t::number_unsigned: + if (JSON_HEDLEY_LIKELY(integer_fits_max(value.m_data.m_value.number_unsigned))) + { + write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); + string_open = false; + break; + } + // too large for BON8: throws + write_bon8_value(value, string_open, depth + 1); + break; + + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + write_bon8_value(value, string_open, depth + 1); + break; + } + } + } + + /*! + @brief write a binary value as BON8 + + BON8 has no binary type: the bytes are written as an array of integers, + like UBJSON and BJData do. Kept out of line, so that write_bon8_value does + not carry the registers of its loop. + */ + JSON_HEDLEY_NEVER_INLINE + void write_bon8_binary(const binary_t& binary, bool& string_open) + { + const auto N = binary.size(); + write_bon8_marker(static_cast(N <= 4 ? 0x80 + N : 0x85), string_open); + + // a byte is an integer of 0 to 255: one byte up to 39, two bytes above + const auto* const data = binary.data(); + for (std::size_t i = 0; i < N;) + { + const std::size_t block = (std::min)(N - i, max_reserve_size / 2); + CharType* const start = oa.reserve(2 * block); + CharType* out = start; + for (std::size_t k = 0; k < block; ++k, ++i) + { + // the cast is needed for binary types whose value type + // is not an integer (e.g., std::byte) + const auto value = static_cast(static_cast(data[i])); + // both bytes are stored, the second only counts for a value above 39: + // 0xC2 + ((value - 40) >> 7), (value - 40) & 0x7F + const unsigned wide = value - 40u; + const bool is_wide = value > 39u; + out[0] = to_char_type(static_cast(is_wide ? 0xC2u + (wide >> 7u) : 0x90u + value)); + out[1] = to_char_type(static_cast(wide & 0x7Fu)); + out += is_wide ? 2 : 1; + } + oa.commit(static_cast(out - start)); + } + + if (N > 4) + { + write_bon8_marker(0xFE, string_open); + } + } + /*! @brief write a single byte that is not part of a string @@ -3328,15 +4827,25 @@ class binary_writer */ static void check_utf8(const string_t& s, const BasicJsonType& context) { - static_cast(context); // only used when exceptions are enabled const auto* data = reinterpret_cast(s.data()); const std::size_t valid = valid_utf8_prefix(data, s.size()); if (JSON_HEDLEY_UNLIKELY(valid != s.size())) { - JSON_THROW(type_error::create(exception_id::invalid_utf8, concat("invalid UTF-8 byte at index ", std::to_string(valid), ": 0x", detail::hex_byte(data[valid])), &context)); + throw_invalid_utf8(data, valid, context); } } + /// @brief throw type_error.316 for the invalid byte at @a index of @a data; + /// out of line, so that the message does not burden check_utf8() + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_invalid_utf8(const unsigned char* data, const std::size_t index, const BasicJsonType& context) + { + static_cast(data); // only used when exceptions are enabled + static_cast(index); + static_cast(context); + JSON_THROW(type_error::create(exception_id::invalid_utf8, concat("invalid UTF-8 byte at index ", std::to_string(index), ": 0x", detail::hex_byte(data[index])), &context)); + } + /*! @brief return @a s as it should be written, honoring @ref error_handler @@ -3585,6 +5094,67 @@ class binary_writer oa.write_characters(vec.data(), sizeof(NumberType)); } + /// whether @a n is written as a float32 by @ref write_compact_float + static bool fits_float32(const number_float_t n) noexcept + { + static_assert(std::is_same::value || std::is_same::value, + "number_float_t must be float or double for the CBOR/MessagePack/BON8 writer"); +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wduplicated-branches") +#endif + return !std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && + static_cast(n) <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(n)) == static_cast(n))); +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + } + + /*! + @brief encode @a n to @a out as @ref write_compact_float writes it + @return the number of characters, 5 or 9 + */ + std::size_t encode_compact_float(CharType* out, const number_float_t n, const CharType float32_marker, const CharType float64_marker) const noexcept + { + if (fits_float32(n)) + { + *out = float32_marker; + store_number(out + 1, static_cast(n), false); + return 1 + sizeof(float); + } + *out = float64_marker; + store_number(out + 1, n, false); + return 1 + sizeof(number_float_t); + } + + /*! + @brief encode a float as @ref write_cbor writes it + @return the number of characters, 3, 5 or 9 + */ + std::size_t encode_cbor_float(CharType* out, const number_float_t n) const noexcept + { + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(n))) + { + // NaN is 0xf97e00, Infinity is 0xf97c00, -Infinity is 0xf9fc00 + out[0] = to_char_type(0xF9); + if (std::isnan(n)) + { + out[1] = to_char_type(0x7E); + } + else + { + out[1] = n > 0 ? to_char_type(0x7C) : to_char_type(0xFC); + } + out[2] = to_char_type(0x00); + return 3; + } + return encode_compact_float(out, n, to_char_type(0xFA), to_char_type(0xFB)); + } + /// @brief write @a n using @a float32_marker if it round-trips through /// float, otherwise using @a float64_marker /// @@ -3594,24 +5164,7 @@ class binary_writer /// no longer needs to be passed in. void write_compact_float(const number_float_t n, const CharType float32_marker, const CharType float64_marker) { - static_assert(std::is_same::value || std::is_same::value, - "number_float_t must be float or double for the CBOR/MessagePack/BON8 writer"); -#ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_PUSH - JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") -#endif - // When number_float_t is float, static_cast(n) is the identity and - // both branches below are intentionally identical (the "compact" float - // representation is the value itself). Only GCC diagnoses this, and only - // when the sink calls are inlined; clang has no such warning. - // (-Wduplicated-branches only exists from GCC 7 on; naming it on an older - // GCC would itself warn under -Wpragmas) -#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) - JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wduplicated-branches") -#endif - if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && - static_cast(n) <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(n)) == static_cast(n)))) + if (fits_float32(n)) { oa.write_character(float32_marker); write_number(static_cast(n)); @@ -3621,9 +5174,6 @@ class binary_writer oa.write_character(float64_marker); write_number(n); } -#ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_POP -#endif } public: diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index f9aa76688..e40d98548 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -55,6 +55,66 @@ using output_adapter_t = std::shared_ptr>; /// output_vector_sink also appends writes of at least this size with insert() constexpr std::size_t max_reserve_size = 256; +/// @brief copy @a length characters, at most 32, without a call of memcpy() +/// +/// Short strings are the typical payload of a binary value, and a call of +/// memcpy() costs more than copying them. A copy of 16 to 32 characters is two +/// possibly overlapping 16-byte copies, one of 8 to 15 two 8-byte copies, and +/// so on down to a single character; each is a fixed-size load and store. +/// Nothing outside of [src, src + length) is read or written. +template +inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::true_type /*is_byte*/) noexcept +{ + JSON_ASSERT(length <= 32); + if (length >= 16) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 16); + std::memcpy(last.data(), src + (length - 16), 16); + std::memcpy(dst, first.data(), 16); + std::memcpy(dst + (length - 16), last.data(), 16); + } + else if (length >= 8) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 8); + std::memcpy(last.data(), src + (length - 8), 8); + std::memcpy(dst, first.data(), 8); + std::memcpy(dst + (length - 8), last.data(), 8); + } + else if (length >= 4) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 4); + std::memcpy(last.data(), src + (length - 4), 4); + std::memcpy(dst, first.data(), 4); + std::memcpy(dst + (length - 4), last.data(), 4); + } + else if (length != 0) + { + // 1 to 3: the first, the middle, and the last character + const CharType a = src[0]; + const CharType b = src[length >> 1]; + const CharType c = src[length - 1]; + dst[0] = a; + dst[length >> 1] = b; + dst[length - 1] = c; + } +} + +/// @brief copy @a length characters, of a character type wider than a byte +template +inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::false_type /*is_byte*/) noexcept +{ + std::memcpy(dst, src, length * sizeof(CharType)); +} + +/// the number of characters an empty vector is grown to by output_vector_sink +constexpr std::size_t min_vector_sink_capacity = 64; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -78,7 +138,7 @@ class output_vector_sink { public: explicit output_vector_sink(std::vector& vec) noexcept - : v(&vec), position(vec.size()) + : v(&vec), cursor(vec.data() + vec.size()), limit(cursor) {} output_vector_sink(const output_vector_sink&) = delete; @@ -86,7 +146,7 @@ class output_vector_sink output_vector_sink& operator=(output_vector_sink&&) = delete; output_vector_sink(output_vector_sink&& other) noexcept - : v(other.v), position(other.position) + : v(other.v), cursor(other.cursor), limit(other.limit) { other.v = nullptr; } @@ -95,7 +155,7 @@ class output_vector_sink { if (v != nullptr) { - v->resize(position); + v->resize(position()); } } @@ -104,11 +164,11 @@ class output_vector_sink JSON_HEDLEY_ALWAYS_INLINE void write_character(CharType c) { - if (JSON_HEDLEY_UNLIKELY(position == v->size())) + if (JSON_HEDLEY_UNLIKELY(cursor == limit)) { grow(1); } - (*v)[position++] = c; + *cursor++ = c; } // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null @@ -117,7 +177,7 @@ class output_vector_sink JSON_HEDLEY_ALWAYS_INLINE void write_characters(const CharType* s, std::size_t length) { - if (JSON_HEDLEY_UNLIKELY(v->size() - position < length)) + if (JSON_HEDLEY_UNLIKELY(space() < length)) { if (length >= max_reserve_size) { @@ -126,10 +186,15 @@ class output_vector_sink } grow(length); } - if (length != 0) + if (length <= 32) { - std::memcpy(v->data() + position, s, length * sizeof(CharType)); - position += length; + copy_short(cursor, s, length, std::integral_constant()); + cursor += length; + } + else + { + std::memcpy(cursor, s, length * sizeof(CharType)); + cursor += length; } } @@ -144,21 +209,37 @@ class output_vector_sink CharType* reserve(const std::size_t length) { JSON_ASSERT(length <= max_reserve_size); - if (JSON_HEDLEY_UNLIKELY(v->size() - position < length)) + if (JSON_HEDLEY_UNLIKELY(space() < length)) { grow(length); } - return v->data() + position; + return cursor; } /// keep the first @a length characters written to the space from reserve() JSON_HEDLEY_ALWAYS_INLINE void commit(const std::size_t length) noexcept { - position += length; + cursor += length; } + /// nothing to pass on: the characters are in the vector (see output_adapter_sink) + void flush() noexcept + {} + private: + /// number of characters of *v written so far + std::size_t position() const noexcept + { + return static_cast(cursor - v->data()); + } + + /// number of characters that fit the vector behind the write position + std::size_t space() const noexcept + { + return static_cast(limit - cursor); + } + /// make room for at least @a length more characters after the position; /// kept out of line so that the write functions stay small enough for /// GCC to inline them into the writer @@ -166,12 +247,15 @@ class output_vector_sink void grow(const std::size_t length) { // use the capacity the vector already has, else at least double it; - // never less than 64 characters, so that a small value does not + // never less than min_vector_sink_capacity, so that a small value does not // reallocate for every few bytes - const std::size_t needed = position + length; - v->resize((std::max)(std::size_t{64}, needed <= v->capacity() + const std::size_t written = position(); + const std::size_t needed = written + length; + v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity() ? v->capacity() : (std::max)(needed, v->capacity() * 2))); + cursor = v->data() + written; + limit = v->data() + v->size(); } /// append @a length (at least max_reserve_size) characters that do not fit @@ -181,20 +265,24 @@ class output_vector_sink JSON_HEDLEY_NEVER_INLINE void append(const CharType* s, const std::size_t length) { - v->resize(position); - if (v->capacity() - position < length) + const std::size_t written = position(); + v->resize(written); + if (v->capacity() - written < length) { // grow geometrically, with room for the writes that follow, so // that the next grow() need not reallocate right away - v->reserve((std::max)(v->capacity() * 2, position + length + max_reserve_size)); + v->reserve((std::max)(v->capacity() * 2, written + length + max_reserve_size)); } v->insert(v->end(), s, s + length); - position = v->size(); + cursor = v->data() + v->size(); + limit = cursor; } std::vector* v; - /// number of characters of *v written so far - std::size_t position; + /// the write position: in *v, at the number of characters written so far + CharType* cursor; + /// the end of *v + CharType* limit; }; /// output adapter for byte vectors @@ -277,9 +365,17 @@ class output_string_adapter : public output_adapter_protocol /// /// Wraps the polymorphic output_adapter_t so the same binary_writer template can /// also target arbitrary adapters (output streams, strings, user-provided -/// adapters) via the `output_adapter`-based overloads. Each write still goes -/// through one virtual call, exactly as before; only the concrete sinks above -/// avoid it. +/// adapters) via the `output_adapter`-based overloads. +/// +/// A write through the adapter is a virtual call, and for a stream a call with +/// a sentry on top, which costs more than encoding the few bytes of most values. +/// The sink therefore collects what is written in a buffer and passes it on in +/// one piece when it is full, a write is too large for it, or @ref flush is +/// called. Everything written reaches the adapter in the same order and with +/// the same bytes; the destructor passes on what is left, so that a write that +/// throws leaves the adapter with exactly the bytes written before it. Only +/// @ref flush can report an error of the adapter (the destructor must not +/// throw), so it has to be called once the value is written. template class output_adapter_sink { @@ -290,36 +386,121 @@ class output_adapter_sink JSON_ASSERT(oa); } + output_adapter_sink(const output_adapter_sink&) = delete; + output_adapter_sink& operator=(const output_adapter_sink&) = delete; + output_adapter_sink& operator=(output_adapter_sink&&) = delete; + + output_adapter_sink(output_adapter_sink&& other) noexcept + : oa(std::move(other.oa)), used(other.used) + { + std::memcpy(buffer.data(), other.buffer.data(), other.used * sizeof(CharType)); + other.used = 0; + } + + ~output_adapter_sink() + { + // a normal return has flushed already, so this is for a write that threw + if (used != 0) + { + JSON_TRY + { + flush(); + } + JSON_CATCH(...) + { + // nothing can be done about an adapter that throws here + } + } + } + + JSON_HEDLEY_ALWAYS_INLINE void write_character(CharType c) { - oa->write_character(c); + if (JSON_HEDLEY_UNLIKELY(used == buffer_size)) + { + flush(); + } + buffer[used++] = c; } // no JSON_HEDLEY_NON_NULL: forwards (null, 0) for empty payloads, exactly as // the type-erased path already did before this sink existed + JSON_HEDLEY_ALWAYS_INLINE void write_characters(const CharType* s, std::size_t length) { - oa->write_characters(s, length); + if (JSON_HEDLEY_UNLIKELY(buffer_size - used < length)) + { + write_large(s, length); + return; + } + if (length <= 32) + { + copy_short(buffer.data() + used, s, length, std::integral_constant()); + used += length; + } + else + { + std::memcpy(buffer.data() + used, s, length * sizeof(CharType)); + used += length; + } } /// space for writing up to @a length characters (see /// output_vector_sink::reserve) - CharType* reserve(const std::size_t length) noexcept + JSON_HEDLEY_ALWAYS_INLINE + CharType* reserve(const std::size_t length) { JSON_ASSERT(length <= max_reserve_size); - static_cast(length); - return reserved.data(); + if (JSON_HEDLEY_UNLIKELY(buffer_size - used < length)) + { + flush(); + } + return buffer.data() + used; } - /// write the first @a length characters written to the space from reserve() - void commit(const std::size_t length) + /// keep the first @a length characters written to the space from reserve() + JSON_HEDLEY_ALWAYS_INLINE + void commit(const std::size_t length) noexcept { - oa->write_characters(reserved.data(), length); + used += length; + } + + /// pass everything written so far on to the adapter + void flush() + { + if (used != 0) + { + // reset first: the characters count as written even if the adapter throws + const std::size_t length = used; + used = 0; + oa->write_characters(buffer.data(), length); + } } private: + /// a write that does not fit the space left in the buffer + JSON_HEDLEY_NEVER_INLINE + void write_large(const CharType* s, const std::size_t length) + { + flush(); + if (length >= buffer_size) + { + oa->write_characters(s, length); + } + else + { + std::memcpy(buffer.data(), s, length * sizeof(CharType)); + used = length; + } + } + + /// large enough for reserve() at any time + static constexpr std::size_t buffer_size = 4 * max_reserve_size; + output_adapter_t oa; - std::array reserved{}; + /// number of characters of @ref buffer not yet passed on + std::size_t used = 0; + std::array buffer{}; }; /// @brief whether std::basic_string has a non-deprecated std::char_traits diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 7bb4edfc5..5eb9906a1 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -71,7 +71,7 @@ class serializer None of @a pretty_print_, @a ensure_ascii_ and @a indent_step_ change over the life of the serializer, so they are captured once here instead of - being threaded through every call to @ref dump, @ref dump_internal and + being threaded through every call to @ref dump, @ref dump_container and @ref dump_iteratively. */ serializer(output_adapter_protocol& s, const char ichar, @@ -80,11 +80,14 @@ class serializer const std::size_t indent_step_ = 0, error_handler_t error_handler_ = error_handler_t::strict) : o(&s) + , number_buffer() // NOLINT(readability-redundant-member-init) + , string_buffer() // NOLINT(readability-redundant-member-init) , indent_char(ichar) , pretty_print(pretty_print_) , ensure_ascii(ensure_ascii_) , indent_step(indent_step_) , error_handler(error_handler_) + , write_buffer() // NOLINT(readability-redundant-member-init) {} // deleted because of pointer members @@ -115,18 +118,18 @@ class serializer void dump(const BasicJsonType& val, const std::size_t current_indent = 0) { - dump_internal(val, current_indent); + dump_element(val, current_indent, 0); flush(); } JSON_PRIVATE_UNLESS_TESTED: /*! - @brief worker for @ref dump + @brief serialize the object or array @a val - Identical in behavior to the historical @ref dump, but writes into the - serializer's internal @ref write_buffer instead of issuing a virtual call - per token. The public @ref dump wraps this and flushes the buffer once the - top-level value has been serialized. + Writes into the serializer's internal @ref write_buffer instead of issuing + a virtual call per token; the public @ref dump flushes the buffer once the + top-level value has been serialized. Scalars are written by @ref dump_scalar, + which @ref dump_element calls without entering this function. Serializing a container descends into its elements, so a value nested deeply enough used to exhaust the call stack and terminate the process with no @@ -137,167 +140,149 @@ class serializer @sa https://github.com/nlohmann/json/issues/5387 */ - void dump_internal(const BasicJsonType& val, - const std::size_t current_indent = 0, - const std::size_t depth = 0) + void dump_container(const BasicJsonType& val, + const std::size_t current_indent, + const std::size_t depth) { - switch (val.m_data.m_type) + JSON_ASSERT(val.m_data.m_type == value_t::object || val.m_data.m_type == value_t::array); + + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) { - case value_t::object: + dump_iteratively(val, current_indent); + return; + } + + if (val.m_data.m_type == value_t::object) + { + if (val.m_data.m_value.object->empty()) { - if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) - { - dump_iteratively(val, current_indent); - return; - } - - if (val.m_data.m_value.object->empty()) - { - put_literal("{}"); - return; - } - - if (pretty_print) - { - put_literal("{\n"); - - // variable to hold indentation for recursive calls - const auto new_indent = next_indent(current_indent, indent_step); - - // first n-1 elements - auto i = val.m_data.m_value.object->cbegin(); - for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) - { - put_indent(new_indent); - put_char('"'); - dump_escaped(i->first); - put_literal("\": "); - dump_internal(i->second, new_indent, depth + 1); - put_literal(",\n"); - } - - // last element - JSON_ASSERT(i != val.m_data.m_value.object->cend()); - JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - put_indent(new_indent); - put_char('"'); - dump_escaped(i->first); - put_literal("\": "); - dump_internal(i->second, new_indent, depth + 1); - - put_char('\n'); - put_indent(current_indent); - put_char('}'); - } - else - { - put_char('{'); - - // first n-1 elements - auto i = val.m_data.m_value.object->cbegin(); - for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) - { - put_char('"'); - dump_escaped(i->first); - put_literal("\":"); - dump_internal(i->second, current_indent, depth + 1); - put_char(','); - } - - // last element - JSON_ASSERT(i != val.m_data.m_value.object->cend()); - JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - put_char('"'); - dump_escaped(i->first); - put_literal("\":"); - dump_internal(i->second, current_indent, depth + 1); - - put_char('}'); - } - + put_literal("{}"); return; } - case value_t::array: + if (pretty_print) { - if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) + // variable to hold indentation for recursive calls + const auto new_indent = next_indent(current_indent, indent_step); + + put_literal_indent("{\n", new_indent); + + // first n-1 elements + auto i = val.m_data.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - dump_iteratively(val, current_indent); - return; + put_object_key(i->first, "\": "); + dump_element(i->second, new_indent, depth + 1); + put_literal_indent(",\n", new_indent); } - if (val.m_data.m_value.array->empty()) + // last element + JSON_ASSERT(i != val.m_data.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); + put_object_key(i->first, "\": "); + dump_element(i->second, new_indent, depth + 1); + + put_literal_indent("\n", current_indent); + put_char('}'); + } + else + { + put_char('{'); + + // first n-1 elements + auto i = val.m_data.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - put_literal("[]"); - return; + put_object_key(i->first, "\":"); + dump_element(i->second, 0, depth + 1); + put_char(','); } - if (pretty_print) - { - put_literal("[\n"); + // last element + JSON_ASSERT(i != val.m_data.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); + put_object_key(i->first, "\":"); + dump_element(i->second, 0, depth + 1); - // variable to hold indentation for recursive calls - const auto new_indent = next_indent(current_indent, indent_step); + put_char('}'); + } + return; + } - // first n-1 elements - for (auto i = val.m_data.m_value.array->cbegin(); - i != val.m_data.m_value.array->cend() - 1; ++i) - { - put_indent(new_indent); - dump_internal(*i, new_indent, depth + 1); - put_literal(",\n"); - } + if (val.m_data.m_value.array->empty()) + { + put_literal("[]"); + return; + } - // last element - JSON_ASSERT(!val.m_data.m_value.array->empty()); - put_indent(new_indent); - dump_internal(val.m_data.m_value.array->back(), new_indent, depth + 1); + if (pretty_print) + { + // variable to hold indentation for recursive calls + const auto new_indent = next_indent(current_indent, indent_step); - put_char('\n'); - put_indent(current_indent); - put_char(']'); - } - else - { - put_char('['); + put_literal_indent("[\n", new_indent); - // first n-1 elements - for (auto i = val.m_data.m_value.array->cbegin(); - i != val.m_data.m_value.array->cend() - 1; ++i) - { - dump_internal(*i, current_indent, depth + 1); - put_char(','); - } - - // last element - JSON_ASSERT(!val.m_data.m_value.array->empty()); - dump_internal(val.m_data.m_value.array->back(), current_indent, depth + 1); - - put_char(']'); - } - - return; + // first n-1 elements + for (auto i = val.m_data.m_value.array->cbegin(); + i != val.m_data.m_value.array->cend() - 1; ++i) + { + dump_element(*i, new_indent, depth + 1); + put_literal_indent(",\n", new_indent); } - case value_t::string: - case value_t::binary: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::discarded: - case value_t::null: - default: - dump_scalar(val, current_indent); - return; + // last element + JSON_ASSERT(!val.m_data.m_value.array->empty()); + dump_element(val.m_data.m_value.array->back(), new_indent, depth + 1); + + put_literal_indent("\n", current_indent); + put_char(']'); + } + else + { + put_char('['); + + // first n-1 elements + for (auto i = val.m_data.m_value.array->cbegin(); + i != val.m_data.m_value.array->cend() - 1; ++i) + { + dump_element(*i, 0, depth + 1); + put_char(','); + } + + // last element + JSON_ASSERT(!val.m_data.m_value.array->empty()); + dump_element(val.m_data.m_value.array->back(), 0, depth + 1); + + put_char(']'); } } private: + /*! + @brief serialize an element of a container + + Most elements are scalars, which need none of the state dump_container() keeps + for a container; writing them from here spares each the entry and exit of + that function. + */ + JSON_HEDLEY_ALWAYS_INLINE void dump_element(const BasicJsonType& val, + const std::size_t current_indent, + const std::size_t depth) + { + if (val.m_data.m_type == value_t::object || val.m_data.m_type == value_t::array) + { + dump_container(val, current_indent, depth); + } + else + { + dump_scalar(val, current_indent); + } + } + /*! @brief write out @a val and everything below it without the call stack - Emits the same bytes as @ref dump_internal, keeping the containers it has + Emits the same bytes as @ref dump_container, keeping the containers it has entered on an explicit stack instead of descending into them. Only reached for values nested deeper than @ref recursion_depth_limit, which is why it is not written for speed: walking every value this way measured up to 20% slower on @@ -325,8 +310,7 @@ class serializer { if (pretty_print) { - put_char('\n'); - put_indent(frame.current_indent); + put_literal_indent("\n", frame.current_indent); } put_char('}'); @@ -353,16 +337,13 @@ class serializer put_indent(frame.child_indent); } - put_char('"'); - dump_escaped(frame.object_it->first); - if (pretty_print) { - put_literal("\": "); + put_object_key(frame.object_it->first, "\": "); } else { - put_literal("\":"); + put_object_key(frame.object_it->first, "\":"); } const BasicJsonType& element = frame.object_it->second; @@ -381,8 +362,7 @@ class serializer { if (pretty_print) { - put_char('\n'); - put_indent(frame.current_indent); + put_literal_indent("\n", frame.current_indent); } put_char(']'); @@ -523,7 +503,7 @@ class serializer /*! @brief serialize the value @a val, which is neither an object nor an array - Shared by @ref dump_internal and @ref dump_value, so that a value is written + Shared by @ref dump_element and @ref dump_value, so that a value is written the same way however deeply it is nested. A binary value is written out here in full: it looks like an object, but has no elements to descend into. @@ -537,78 +517,13 @@ class serializer { case value_t::string: { - put_char('"'); - dump_escaped(*val.m_data.m_value.string); - put_char('"'); + dump_string_value(*val.m_data.m_value.string); return; } case value_t::binary: { - if (pretty_print) - { - put_literal("{\n"); - - // variable to hold indentation for the bytes - const auto new_indent = next_indent(current_indent, indent_step); - - put_indent(new_indent); - - put_literal("\"bytes\": ["); - - if (!val.m_data.m_value.binary->empty()) - { - for (auto i = val.m_data.m_value.binary->cbegin(); - i != val.m_data.m_value.binary->cend() - 1; ++i) - { - dump_byte(*i); - put_literal(", "); - } - dump_byte(val.m_data.m_value.binary->back()); - } - - put_literal("],\n"); - put_indent(new_indent); - - put_literal("\"subtype\": "); - if (val.m_data.m_value.binary->has_subtype()) - { - dump_integer(val.m_data.m_value.binary->subtype()); - } - else - { - put_literal("null"); - } - put_char('\n'); - put_indent(current_indent); - put_char('}'); - } - else - { - put_literal("{\"bytes\":["); - - if (!val.m_data.m_value.binary->empty()) - { - for (auto i = val.m_data.m_value.binary->cbegin(); - i != val.m_data.m_value.binary->cend() - 1; ++i) - { - dump_byte(*i); - put_char(','); - } - dump_byte(val.m_data.m_value.binary->back()); - } - - put_literal("],\"subtype\":"); - if (val.m_data.m_value.binary->has_subtype()) - { - dump_integer(val.m_data.m_value.binary->subtype()); - put_char('}'); - } - else - { - put_literal("null}"); - } - } + dump_binary_value(current_indent, *val.m_data.m_value.binary); return; } @@ -627,19 +542,19 @@ class serializer case value_t::number_integer: { - dump_integer(val.m_data.m_value.number_integer); + dump_signed_value(val.m_data.m_value.number_integer); return; } case value_t::number_unsigned: { - dump_integer(val.m_data.m_value.number_unsigned); + dump_unsigned_value(val.m_data.m_value.number_unsigned); return; } case value_t::number_float: { - dump_float(val.m_data.m_value.number_float); + dump_float_value(val.m_data.m_value.number_float); return; } @@ -662,6 +577,100 @@ class serializer } } + /// @brief write the string @a s in quotes; kept out of line, so that the + /// other scalars do not pay for its register needs (and the other way round) + JSON_HEDLEY_NEVER_INLINE void dump_string_value(const string_t& s) + { +#if JSON_LITTLE_ENDIAN_TARGET + if (s.size() <= short_string_max && put_short_quoted(s, "\"")) + { + return; + } +#endif + put_char('"'); + dump_escaped(s); + put_char('"'); + } + + // The numbers, and the rest of what dump_scalar() writes, are left to + // functions of their own as well: a single function holding all of them + // saves and restores every register any of them needs, on every call. Now + // dump_scalar() only dispatches, and each writer pays for what it uses. + JSON_HEDLEY_NEVER_INLINE void dump_signed_value(const number_integer_t x) + { + dump_integer(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_unsigned_value(const number_unsigned_t x) + { + dump_integer(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_float_value(const number_float_t x) + { + dump_float(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_binary_value(const std::size_t current_indent, const typename BasicJsonType::binary_t& binary) + { + if (pretty_print) + { + // variable to hold indentation for the bytes + const auto new_indent = next_indent(current_indent, indent_step); + + put_literal_indent("{\n", new_indent); + put_literal("\"bytes\": ["); + + if (!binary.empty()) + { + for (auto i = binary.cbegin(); i != binary.cend() - 1; ++i) + { + dump_byte(*i); + put_literal(", "); + } + dump_byte(binary.back()); + } + + put_literal_indent("],\n", new_indent); + + put_literal("\"subtype\": "); + if (binary.has_subtype()) + { + dump_integer(binary.subtype()); + } + else + { + put_literal("null"); + } + put_literal_indent("\n", current_indent); + put_char('}'); + } + else + { + put_literal("{\"bytes\":["); + + if (!binary.empty()) + { + for (auto i = binary.cbegin(); i != binary.cend() - 1; ++i) + { + dump_byte(*i); + put_char(','); + } + dump_byte(binary.back()); + } + + put_literal("],\"subtype\":"); + if (binary.has_subtype()) + { + dump_integer(binary.subtype()); + put_char('}'); + } + else + { + put_literal("null}"); + } + } + } /*! @brief the indentation level to use for the children of the current value @@ -678,6 +687,220 @@ class serializer return new_indent; } + private: + /*! + @brief append the object key @a s in quotes, followed by @a tail + + @a tail is the closing quote and the colon, followed by a space when + pretty-printing. The callers pass it because each of them already sits on + the pretty-printing or the compact path; choosing it here would test + @ref pretty_print once more for every key. Most keys are short and plain + and are written in one go by put_short_quoted(); everything else takes the + general path. + */ + template + void put_object_key(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { +#if JSON_LITTLE_ENDIAN_TARGET + if (put_short_quoted(s, tail)) + { + return; + } +#endif + put_char('"'); + dump_escaped(s); + put_literal(tail); + } + + // put_short_quoted() assembles words from little-endian loads +#if JSON_LITTLE_ENDIAN_TARGET + /// the longest string put_short_quoted() takes + static constexpr std::size_t short_string_max = JSON_SCAN_VECTOR ? 32 : 16; + + /// the high bit of every byte of @a v that is not printable ASCII or is a + /// quote, a backslash, or DEL (zero if the bytes can be copied as they are) + static std::uint64_t swar_not_plain(const std::uint64_t v) noexcept + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' + return (((q - ones) & ~q) | ((b - ones) & ~b) // == '"', == '\\' + | ((v - 0x2020202020202020ull) & ~v) // < 0x20 + | v | (v + ones)) // >= 0x7F + & 0x8080808080808080ull; + } + + template + static WordType load_word(const char* p) noexcept + { + WordType w; + std::memcpy(&w, p, sizeof(w)); + return w; + } + + template + static void store_word(char* p, const WordType w) noexcept + { + std::memcpy(p, &w, sizeof(w)); + } + + /*! + @brief write `"`, @a s and @a tail if @a s is short and needs no escaping + + A string of up to 16 (32 with vector instructions) printable ASCII + characters (no quote, backslash, or DEL) is read with two overlapping loads + and written with two overlapping stores, instead of scanning it and calling + memcpy with a length that is only known at run time. Returns false, having + changed nothing, if @a s does not qualify or the write buffer lacks the + room. + */ + template + bool put_short_quoted(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + const std::size_t n = s.size(); + if (n > short_string_max || write_buffer_pos + n + N > write_buffer.size()) + { + return false; + } + + char* d = write_buffer.data() + write_buffer_pos + 1; + const char* const p = s.data(); + + if (n >= 8) + { +#if JSON_SCAN_VECTOR + if (!(n >= 16 ? copy_plain_vectors(d, p, n) : copy_plain_halves(d, p, n))) + { + return false; + } +#else + const auto a = load_word(p); + const auto b = load_word(p + n - 8); + if ((swar_not_plain(a) | swar_not_plain(b)) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 8, b); +#endif + } + else if (n >= 4) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 4); + // the bytes above the string are filled with a plain character + const std::uint64_t w = a | (static_cast(b) << (8 * (n - 4))) + | (0x4141414141414141ull << (8 * n)); + if (swar_not_plain(w) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 4, b); + } + else if (n >= 2) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 2); + const std::uint64_t w = a | (static_cast(b) << (8 * (n - 2))) + | (0x4141414141414141ull << (8 * n)); + if (swar_not_plain(w) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 2, b); + } + else if (n == 1) + { + const std::uint64_t w = static_cast(static_cast(p[0])) | (0x4141414141414141ull << 8u); + if (swar_not_plain(w) != 0) + { + return false; + } + *d = p[0]; + } + + d[-1] = '"'; + std::memcpy(d + n, tail, N - 1); + write_buffer_pos += n + N; + return true; + } + +#if JSON_SCAN_VECTOR + /// copy the 8 to 15 bytes [@a p, @a p + @a n) to @a d with two overlapping + /// words, tested together as one vector, see copy_plain_vectors() + static bool copy_plain_halves(char* d, const char* p, const std::size_t n) noexcept + { +#if JSON_SCAN_NEON + const uint8x8_t lo = vld1_u8(reinterpret_cast(p)); + const uint8x8_t hi = vld1_u8(reinterpret_cast(p + n - 8)); + const uint8x16_t in = vcombine_u8(lo, hi); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)), vceqq_u8(in, vdupq_n_u8(0x7F)))); + if (vmaxvq_u8(special) != 0) + { + return false; + } + vst1_u8(reinterpret_cast(d), lo); + vst1_u8(reinterpret_cast(d + n - 8), hi); +#else + const __m128i lo = _mm_loadl_epi64(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i hi = _mm_loadl_epi64(static_cast(static_cast(p + n - 8))); // NOLINT(bugprone-casting-through-void) + const __m128i in = _mm_unpacklo_epi64(lo, hi); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_or_si128(_mm_cmplt_epi8(in, _mm_set1_epi8(0x20)), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)))); + if (_mm_movemask_epi8(special) != 0) + { + return false; + } + _mm_storel_epi64(static_cast<__m128i*>(static_cast(d)), lo); // NOLINT(bugprone-casting-through-void) + _mm_storel_epi64(static_cast<__m128i*>(static_cast(d + n - 8)), hi); // NOLINT(bugprone-casting-through-void) +#endif + return true; + } + + /// copy the 16 to 32 bytes [@a p, @a p + @a n) to @a d with two overlapping + /// vectors, unless one of them is a quote, a backslash, a control + /// character, DEL or not ASCII + static bool copy_plain_vectors(char* d, const char* p, const std::size_t n) noexcept + { +#if JSON_SCAN_NEON + const uint8x16_t a = vld1q_u8(reinterpret_cast(p)); + const uint8x16_t b = vld1q_u8(reinterpret_cast(p + n - 16)); + const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('"')), vceqq_u8(a, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(b, vdupq_n_u8('"')), vceqq_u8(b, vdupq_n_u8('\\')))), + vorrq_u8(vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(a), vdupq_n_s8(0x20)), vcltq_s8(vreinterpretq_s8_u8(b), vdupq_n_s8(0x20))), + vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F))))); + if (vmaxvq_u8(special) != 0) + { + return false; + } + vst1q_u8(reinterpret_cast(d), a); + vst1q_u8(reinterpret_cast(d + n - 16), b); +#else + const __m128i a = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i b = _mm_loadu_si128(static_cast(static_cast(p + n - 16))); // NOLINT(bugprone-casting-through-void) + const __m128i quote = _mm_set1_epi8('"'); + const __m128i backslash = _mm_set1_epi8('\\'); + const __m128i space = _mm_set1_epi8(0x20); + const __m128i del = _mm_set1_epi8(0x7F); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, quote), _mm_cmpeq_epi8(a, backslash)), + _mm_or_si128(_mm_cmpeq_epi8(b, quote), _mm_cmpeq_epi8(b, backslash))), + _mm_or_si128(_mm_or_si128(_mm_cmplt_epi8(a, space), _mm_cmplt_epi8(b, space)), + _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del)))); + if (_mm_movemask_epi8(special) != 0) + { + return false; + } + _mm_storeu_si128(static_cast<__m128i*>(static_cast(d)), a); // NOLINT(bugprone-casting-through-void) + _mm_storeu_si128(static_cast<__m128i*>(static_cast(d + n - 16)), b); // NOLINT(bugprone-casting-through-void) +#endif + return true; + } +#endif +#endif + JSON_PRIVATE_UNLESS_TESTED: /*! @brief dump escaped string @@ -690,14 +913,23 @@ class serializer @param[in] s the string to escape Complexity: Linear in the length of string @a s. + + @note kept out of line: with the short-string attempts in front of it gone + from here, compilers would otherwise inline it into @ref put_object_key + and then no longer into the loops that call that */ - void dump_escaped(const string_t& s) + JSON_HEDLEY_NEVER_INLINE void dump_escaped(const string_t& s) { // Short strings of printable ASCII - most object keys and many values - // are checked byte by byte against a table and then copied in one go. // For them, this is cheaper than starting the vector scanner below, // whose setup costs more than such a string takes to scan. The same // check serves both values of ensure_ascii. + // + // Where put_short_quoted() exists and takes strings of up to 32 bytes, + // the callers have tried it already, and repeating the attempt only + // costs time for strings that need escaping or are not ASCII. +#if !JSON_LITTLE_ENDIAN_TARGET || !JSON_SCAN_VECTOR if (s.size() <= 32) { const auto& copyable = ascii_copyable(); @@ -719,6 +951,7 @@ class serializer return; } } +#endif // dispatch once here rather than test the flag inside the loop: it does // not change while a string is written, and folding it lets each of the @@ -796,7 +1029,7 @@ class serializer put_buffer(string_buffer, bytes); bytes = 0; } - put_string(s, i, i + run); + put_run(s, i, run); bytes_after_last_accept = 0; undumped_chars = 0; i += run; @@ -836,6 +1069,27 @@ class serializer const auto byte = static_cast(s[i]); + if (EnsureAscii && state == UTF8_ACCEPT && byte >= 0x80) + { + // a run of well-formed multi-byte characters is escaped + // straight into the write buffer; an ill-formed sequence + // ends the run and is left to the byte-at-a-time path + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + bytes_after_last_accept = 0; + } + const std::size_t escaped = write_u_escapes(reinterpret_cast(s.data()) + i, s.size() - i); + if (escaped != 0) + { + bytes_after_last_accept = 0; + undumped_chars = 0; + i += escaped - 1; + continue; + } + } + switch (decode(state, codepoint, byte)) { case UTF8_ACCEPT: // decode found a new code point @@ -1113,6 +1367,12 @@ class serializer return; } + // the usual indentation, without a call of memset + if (JSON_HEDLEY_LIKELY(put_short_literal_indent("", indent))) + { + return; + } + const std::size_t capacity = write_buffer.size(); // fill whatever room is left in the buffer; this is the whole job @@ -1146,6 +1406,51 @@ class serializer write_buffer_pos = indent; } + /*! + @brief append a string literal and then @a indent indentation characters + + The usual line break and indentation go into the buffer with one check of + its room and four stores of the indentation, see @ref put_short_literal_indent. + */ + template + JSON_HEDLEY_ALWAYS_INLINE void put_literal_indent(const char (&s)[N], const std::size_t indent) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + if (JSON_HEDLEY_LIKELY(put_short_literal_indent(s, indent))) + { + return; + } + put_literal(s); + put_indent(indent); + } + + /*! + @brief append a string literal and up to 32 indentation characters in one go + + Four words of the indentation character are stored unconditionally and only + the part wanted is claimed, which spares the call of memset. Shared by + @ref put_indent (with an empty literal, whose copy compiles to nothing) and + @ref put_literal_indent. Returns false, having written nothing, if + @a indent is wider or the write buffer lacks the room. + */ + template + JSON_HEDLEY_ALWAYS_INLINE bool put_short_literal_indent(const char (&s)[N], const std::size_t indent) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + constexpr std::size_t length = N - 1; + if (indent > 32 || write_buffer_pos + length + 32 > write_buffer.size()) + { + return false; + } + const std::uint64_t word = 0x0101010101010101ull * static_cast(indent_char); + char* const d = write_buffer.data() + write_buffer_pos; + std::memcpy(d, s, length); + std::memcpy(d + length, &word, sizeof(word)); + std::memcpy(d + length + 8, &word, sizeof(word)); + std::memcpy(d + length + 16, &word, sizeof(word)); + std::memcpy(d + length + 24, &word, sizeof(word)); + write_buffer_pos += length + indent; + return true; + } + /*! @brief append a string literal to the write buffer @@ -1170,6 +1475,28 @@ class serializer write_buffer_pos += length; } + /*! + @brief append the characters of @a str in [@a start, @a start + @a length) + + A run of up to 16 characters with 16 or more in the string from @a start on + and in the write buffer from its end on is copied as a whole block of 16, + which spares the call of memcpy with a length known only at run time; the + surplus characters in the buffer are overwritten by what follows. + */ + template + void put_run(const StringType& str, const std::size_t start, const std::size_t length) + { + // a short run is the exception in text without escapes (CJK, accented + // Latin), for which this keeps the long path the fast one + if (JSON_HEDLEY_UNLIKELY(length <= 16 && start + 16 <= str.size() && write_buffer_pos + 16 <= write_buffer.size())) + { + std::memcpy(write_buffer.data() + write_buffer_pos, str.data() + start, 16); + write_buffer_pos += length; + return; + } + put_string(str, start, start + length); + } + /*! @brief append the characters of @a str in [@a start, @a end) @@ -1412,6 +1739,78 @@ class serializer return escapes; } + /*! + @brief escape the well-formed multi-byte characters at the start of @a data + + Writes the "\\uXXXX" escape of each character (two of them for a character + above U+FFFF) straight into the write buffer, which is why the caller must + have written out what is pending in @ref string_buffer. Stops at the first + byte that is below 0x80 or starts an ill-formed or truncated sequence, which + the caller handles. + + @return the number of bytes of @a data that have been escaped + */ + JSON_HEDLEY_NEVER_INLINE std::size_t write_u_escapes(const unsigned char* data, std::size_t n) + { + constexpr const char* nibble_to_hex = "0123456789abcdef"; + std::size_t i = 0; + while (i < n && data[i] >= 0x80) + { + const std::size_t length = validate_one_utf8(data + i, n - i); + if (length == 0) + { + break; + } + + // a lead byte of 0x80 or above starts a sequence of 2 to 4 bytes + JSON_ASSERT(length >= 2 && length <= 4); + + const auto b0 = static_cast(data[i]); + const auto b1 = static_cast(data[i + 1]); + std::uint32_t codepoint = 0; + switch (length) + { + case 2: + codepoint = ((b0 & 0x1Fu) << 6u) | (b1 & 0x3Fu); + break; + case 3: + codepoint = ((b0 & 0x0Fu) << 12u) | ((b1 & 0x3Fu) << 6u) | (static_cast(data[i + 2]) & 0x3Fu); + break; + default: // 4 + codepoint = ((b0 & 0x07u) << 18u) | ((b1 & 0x3Fu) << 12u) | ((static_cast(data[i + 2]) & 0x3Fu) << 6u) | (static_cast(data[i + 3]) & 0x3Fu); + break; + } + + // at most two escapes of six bytes each + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < 12)) + { + flush(); + } + char* d = write_buffer.data() + write_buffer_pos; + if (codepoint > 0xFFFFu) + { + d = put_u_escape(d, static_cast(0xD7C0u + (codepoint >> 10u)), nibble_to_hex); + codepoint = 0xDC00u + (codepoint & 0x3FFu); + } + d = put_u_escape(d, static_cast(codepoint), nibble_to_hex); + write_buffer_pos = static_cast(d - write_buffer.data()); + i += length; + } + return i; + } + + static char* put_u_escape(char* d, const std::uint16_t codeunit, const char* nibble_to_hex) noexcept + { + const auto unit = static_cast(codeunit); + d[0] = '\\'; + d[1] = 'u'; + d[2] = nibble_to_hex[(unit >> 12u) & 0x0Fu]; + d[3] = nibble_to_hex[(unit >> 8u) & 0x0Fu]; + d[4] = nibble_to_hex[(unit >> 4u) & 0x0Fu]; + d[5] = nibble_to_hex[unit & 0x0Fu]; + return d + 6; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * @@ -1769,11 +2168,22 @@ class serializer /// the output of the serializer (non-owning; the adapter lives at the call site) output_adapter_protocol* o = nullptr; + /// a character buffer left uninitialized: the buffers below are written + /// before they are read (only the part written is ever read), and zeroing + /// 1.6 KiB took more than half of what dumping a small value costs (the + /// user-provided constructor keeps the member initialization, which + /// -Weffc++ asks for, from zeroing them) + template + struct uninitialized_buffer : std::array + { + uninitialized_buffer() noexcept {} // NOLINT(modernize-use-equals-default,hicpp-use-equals-default,cppcoreguidelines-pro-type-member-init,hicpp-member-init): see above + }; + /// a (hopefully) large enough character buffer - std::array number_buffer{{}}; + uninitialized_buffer<64> number_buffer; /// string buffer - std::array string_buffer{{}}; + uninitialized_buffer<512> string_buffer; /// the indentation character const char indent_char; @@ -1793,7 +2203,7 @@ class serializer /// buffer collecting output before it is flushed to the output adapter, so /// that the many small structural writes become few bulk writes static constexpr std::size_t write_buffer_size = 1024; - std::array write_buffer{{}}; + uninitialized_buffer write_buffer; /// number of valid bytes currently held in @ref write_buffer std::size_t write_buffer_pos = 0; }; diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 3eb85ce47..3e4ea268a 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -275,11 +275,34 @@ class builder 0, 0, '\b', 0, 0, 0, '\f', 0, 0, 0, 0, 0, 0, 0, '\n', 0, 0, 0, '\r', 0, '\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + // The write position is kept in locals: the stores of characters into + // the arena may alias the members of the builder, which would then be + // reloaded after every one. It is stored back where the arena grows, + // and by `write_back` on every return. + const unsigned char* const end = m_end; const std::size_t start = arena_used(); - arena_run(s, static_cast(p - s)); + char* w = m_arena_write; + const char* w_end = m_arena_end; + const arena_write_back write_back(*this, w); + { + const auto n = static_cast(p - s); + if (n <= 16 && end - s >= 16) + { + arena_ensure(w, w_end, 16); + std::memcpy(w, s, 16); + } + else if (n != 0) + { + // (an empty run leaves an empty arena unallocated, and memcpy() + // must not be passed its null pointer even for zero bytes) + arena_ensure(w, w_end, n); + std::memcpy(w, s, n); + } + w += n; + } for (;;) { - if (p == m_end) + if (p == end) { return failed(error_code::string_missing_quote, p); } @@ -287,7 +310,9 @@ class builder if (c == '"') { ++p; - return decoded{p, start, arena_used() - start}; + // (from w: `write_back` stores it only after the return value + // is made; the arena is allocated, since an escape was written) + return decoded{p, start, static_cast(w - m_doc.arena.data()) - start}; } if (c != '\\') { @@ -297,7 +322,7 @@ class builder return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); } ++p; - if (p == m_end) + if (p == end) { return failed(error_code::string_missing_quote, p); } @@ -315,7 +340,7 @@ class builder { return failed(error_code::string_surrogate_low, p); } - if (m_end - p < 2 || p[0] != '\\' || p[1] != 'u') + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') { return failed(error_code::string_surrogate_high, p); } @@ -331,26 +356,106 @@ class builder } cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); } - arena_ensure(4); - m_arena_write = put_utf8(m_arena_write, cp); + arena_ensure(w, w_end, 4); + w = put_utf8(w, cp); } else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0)) { - arena_ensure(1); - *m_arena_write++ = simple_escape[d]; + arena_ensure(w, w_end, 1); + *w++ = simple_escape[d]; } else { --p; return failed(error_code::string_escape, p); } - if (p != m_end && *p == '\\') +#if NLOHMANN_VIEW_NEON + if (p != end && *p == '\\') { continue; // consecutive escapes ("\u00e4\u00f6"): no run in between } + // The first 16 bytes of the run are checked one by one, in predicted branches, and copied + // as one block: where the runs between escapes are short, the vector step below would put + // its longer dependency chain (load, compare, mask, count, add) on p for each of them, + // and be slower than these steps (the scanner of the keys works the same way) + { + if (end - p >= 16) + { + const std::uint8_t* const plain = string_plain(); + std::size_t steps = 16; +#define NLOHMANN_VIEW_STEP(i) if (NLOHMANN_VIEW_LIKELY(plain[p[i]] != 0)) {} else { steps = (i); goto run_found; } + NLOHMANN_VIEW_REPEAT16(NLOHMANN_VIEW_STEP) +#undef NLOHMANN_VIEW_STEP +run_found: + arena_ensure(w, w_end, 16); + std::memcpy(w, p, 16); // (only the first `steps` bytes are kept) + w += steps; + p += steps; + if (steps < 16) + { + if (*p >= 0x80) + { + // non-ASCII: the UTF-8 check, and the rest of the run + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + arena_ensure(w, w_end, n); + std::memcpy(w, r, n); + w += n; + } + continue; + } + } + } +#else + if (p != end && (*p == '\\' || *p == '"')) + { + continue; // consecutive escapes ("\u00e4\u00f6"), or the end of the string: no run in between + } +#endif +#if NLOHMANN_VIEW_VECTOR + // the plain bytes up to the next quote, backslash, or control + // character are copied as they are scanned, 16 bytes at a time + for (;;) + { + if (NLOHMANN_VIEW_UNLIKELY(end - p < 16)) + { + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + arena_ensure(w, w_end, n); + std::memcpy(w, r, n); + w += n; + break; + } + arena_ensure(w, w_end, 16); + const unsigned k = copy_plain_run16(p, w); + w += k; + p += k; + if (k == 16) + { + continue; + } + if (NLOHMANN_VIEW_UNLIKELY(*p >= 0x80)) + { + // non-ASCII: the UTF-8 check, and the rest of the run + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + arena_ensure(w, w_end, n); + std::memcpy(w, r, n); + w += n; + } + break; + } +#else const unsigned char* const r = p; - p = scan_string_run(p, m_end); - arena_run(r, static_cast(p - r)); + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + arena_ensure(w, w_end, n); + std::memcpy(w, r, n); + w += n; +#endif } } @@ -392,19 +497,49 @@ class builder char* m_arena_write = nullptr; char* m_arena_end = nullptr; + /// room for n more bytes at w, the write position kept in a local, with + /// w_end the end of the arena. (A lambda instead is not always inlined, + /// and then w would be kept in memory, as m_arena_write is.) + NLOHMANN_VIEW_ALWAYS_INLINE void arena_ensure(char*& w, const char*& w_end, std::size_t n) + { + if (NLOHMANN_VIEW_UNLIKELY(static_cast(w_end - w) < n)) + { + m_arena_write = w; + arena_grow(n); + w = m_arena_write; + w_end = m_arena_end; + } + } + + /// stores a write position kept in a local back into m_arena_write when + /// the function that keeps it returns, on every path + class arena_write_back + { + public: + arena_write_back(builder& b, char*& w) noexcept + : m_builder(b) + , m_w(w) + {} + ~arena_write_back() + { + m_builder.m_arena_write = m_w; + } + + arena_write_back(const arena_write_back&) = delete; + arena_write_back& operator=(const arena_write_back&) = delete; + arena_write_back(arena_write_back&&) = delete; + arena_write_back& operator=(arena_write_back&&) = delete; + + private: + builder& m_builder; + char*& m_w; + }; + NLOHMANN_VIEW_ALWAYS_INLINE std::size_t arena_used() const noexcept { return m_arena_write != nullptr ? static_cast(m_arena_write - m_doc.arena.data()) : 0; } - NLOHMANN_VIEW_ALWAYS_INLINE void arena_ensure(std::size_t n) - { - if (NLOHMANN_VIEW_UNLIKELY(static_cast(m_arena_end - m_arena_write) < n)) - { - arena_grow(n); - } - } - NLOHMANN_VIEW_NOINLINE void arena_grow(std::size_t n) { const std::size_t used = arena_used(); @@ -413,25 +548,6 @@ class builder m_arena_end = m_doc.arena.data() + m_doc.arena.size(); } - /// append the run [r, r + n) to the arena; short runs as one fixed-size - /// 16-byte move when both sides have the room (no library call) - NLOHMANN_VIEW_ALWAYS_INLINE void arena_run(const unsigned char* r, std::size_t n) - { - if (n <= 16 && m_end - r >= 16) - { - arena_ensure(16); - std::memcpy(m_arena_write, r, 16); - } - else if (n != 0) - { - // (an empty run leaves an empty arena unallocated, and memcpy() - // must not be passed its null pointer even for zero bytes) - arena_ensure(n); - std::memcpy(m_arena_write, r, n); - } - m_arena_write += n; - } - /// UTF-8 encoding of cp at w (room for 4 bytes); returns the position after /// it. (A returned pointer stays in a register; writing through a /// reference to m_arena_write would store and reload it around every byte.) @@ -954,11 +1070,11 @@ indent_done: ++p; } const unsigned char* const int_start = p; - if (p != e && *p == '0') + if (cur() == '0') { ++p; } - else if (NLOHMANN_VIEW_LIKELY(p != e && *p >= '1' && *p <= '9')) + else if (NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) { p = skip_digits(p + 1, e); } @@ -969,7 +1085,7 @@ indent_done: const auto int_digits = static_cast(p - int_start); std::size_t frac_digits = 0; bool is_float = false; - if (p != e && *p == '.') + if (cur() == '.') { ++p; const unsigned char* const f0 = p; @@ -982,20 +1098,20 @@ indent_done: is_float = true; } std::int64_t exponent = 0; - if (p != e && (*p | 0x20) == 'e') + if ((cur() | 0x20) == 'e') { ++p; bool exp_negative = false; - if (p != e && (*p == '+' || *p == '-')) + if (cur() == '+' || cur() == '-') { exp_negative = *p == '-'; ++p; } - if (NLOHMANN_VIEW_UNLIKELY(p == e || !is_digit(*p))) + if (NLOHMANN_VIEW_UNLIKELY(!is_digit(cur()))) { return fail(error_code::number_after_exponent); } - while (p != e && is_digit(*p)) + while (is_digit(cur())) { if (exponent < 100000) { @@ -1057,7 +1173,7 @@ indent_done: ++p; // opening quote const unsigned char* const s = p; p = scan_string_run(p, e); - if (NLOHMANN_VIEW_LIKELY(p != e && *p == '"')) + if (NLOHMANN_VIEW_LIKELY(cur() == '"')) { push_node(value_t::string, 0, 0, static_cast(s - b), static_cast(p - s)); ++p; diff --git a/include/nlohmann/detail/view/input.hpp b/include/nlohmann/detail/view/input.hpp index 7e1fea09a..d028e81e2 100644 --- a/include/nlohmann/detail/view/input.hpp +++ b/include/nlohmann/detail/view/input.hpp @@ -10,6 +10,9 @@ #include // max #include // size_t +#ifndef JSON_NO_IO + #include // FILE, fread +#endif #include // istream, streambuf #include // basic_string, char_traits, string #include // decay, integral_constant, is_array, is_const, is_integral, is_lvalue_reference, is_pointer, is_same, remove_cv, remove_extent, remove_pointer, remove_reference @@ -154,6 +157,37 @@ inline void append_stream(std::string& buf, std::istream& is) is.clear(is.rdstate() | std::ios::eofbit); } +/// a pointer to a C stream (never with JSON_NO_IO, which leaves out ) +template +struct is_c_stream +#ifndef JSON_NO_IO + : std::is_same +#else + : std::false_type +#endif +{}; + +#ifndef JSON_NO_IO +/// append the rest of a C stream: in blocks with fread, not character by +/// character (the library's file adapter reads with fgetc); +/// f is not null +inline void append_file(std::string& buf, std::FILE* f) +{ + for (;;) + { + const std::size_t used = buf.size(); + const std::size_t room = (std::max)(buf.capacity() - used, std::size_t{4096}); + buf.resize(used + room); + const std::size_t got = std::fread(&buf[used], 1, room, f); + buf.resize(used + got); + if (got < room) + { + break; // end of file, or an error (as fgetc, which returns EOF for both) + } + } +} +#endif + } // namespace view } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/view/iterator.hpp b/include/nlohmann/detail/view/iterator.hpp index 12906e9c1..069f50fd1 100644 --- a/include/nlohmann/detail/view/iterator.hpp +++ b/include/nlohmann/detail/view/iterator.hpp @@ -147,20 +147,21 @@ class view_item view_item(const iterator& it, std::size_t index) : m_it(it) - { - if (!it.is_object_iterator()) - { - m_index = std::to_string(index); - } - } + , m_position(index) + {} - /// the member key, or the element index for arrays + /// the member key, or the element index for arrays (as text made on the + /// first call, so that iterating over the values pays nothing for it) string_view_t key() const { if (m_it.is_object_iterator()) { return m_it.key(); } + if (m_index.empty()) + { + m_index = std::to_string(m_position); + } return string_view_t(m_index.data(), m_index.size()); } @@ -183,7 +184,8 @@ class view_item private: iterator m_it; - std::string m_index{}; // NOLINT(readability-redundant-member-init) + std::size_t m_position; ///< the index of an array element + mutable std::string m_index{}; ///< the text of m_position, once key() was called // NOLINT(readability-redundant-member-init) }; /// the range returned by basic_json_view::items() diff --git a/include/nlohmann/detail/view/lookup.hpp b/include/nlohmann/detail/view/lookup.hpp index 10f7ab400..e2038a956 100644 --- a/include/nlohmann/detail/view/lookup.hpp +++ b/include/nlohmann/detail/view/lookup.hpp @@ -166,6 +166,10 @@ const node* element_at(const document_data& d, const node* array, std::size_t id { return e + idx; // a growable block: one link per element } + if (array->next == array->len + 1u && (!nav::editable() || (array->flags & node_flags::moved) == 0)) + { + return e + idx; // no element has a subtree: the elements are the nodes after the array + } for (std::size_t i = 0; i < idx; ++i) { e = document_data::after(e); @@ -181,6 +185,10 @@ const node* last_child(const document_data& d, const node* container) noexcept const std::size_t value_offset = container->kind == static_cast(value_t::object) ? 1 : 0; const node* const end = navigation::end(d, container); const node* last = navigation::first(d, container); + if (value_offset == 0 && container->next == container->len + 1u && (!navigation::editable() || (container->flags & node_flags::moved) == 0)) + { + return last + (container->len - 1u); // an array without subtrees: the elements are the nodes after it + } for (const node* c = document_data::after(last + value_offset); c != end; c = document_data::after(c + value_offset)) { last = c; diff --git a/include/nlohmann/detail/view/materialize.hpp b/include/nlohmann/detail/view/materialize.hpp index b15bec551..62adb30a9 100644 --- a/include/nlohmann/detail/view/materialize.hpp +++ b/include/nlohmann/detail/view/materialize.hpp @@ -8,7 +8,7 @@ #pragma once -#include // max +#include // array #include // size_t #include // int64_t, uint8_t #include // string @@ -40,12 +40,13 @@ template BasicJsonType materialize(const document_data& d, const node* n) { using string_t = typename BasicJsonType::string_t; - using sax_t = json_sax_dom_parser>; + using sax_t = json_sax_dom_parser, true>; using nav = navigation; // the open containers: their next element (or key of the next member), - // their end, and whether they are objects; sized once from the - // document's deepest nesting (for a subtree an upper bound) + // their end, and whether they are objects; for the first 16 levels on the + // stack, else sized once from the document's deepest nesting (for a + // subtree an upper bound) struct frame { const node* pos; @@ -53,10 +54,43 @@ BasicJsonType materialize(const document_data& d, const node* n) bool object; }; + // a scalar needs neither the handler nor the stack of open containers + switch (static_cast(n->kind)) + { + case value_t::string: + return BasicJsonType(string_t(d.str(*n), n->len)); + case value_t::number_integer: + return BasicJsonType(static_cast(static_cast(integer_bits(*n)))); + case value_t::number_unsigned: + return BasicJsonType(static_cast(integer_bits(*n))); + case value_t::number_float: + return BasicJsonType(float_value(d, *n)); + case value_t::boolean: + return BasicJsonType((n->flags & node_flags::is_true) != 0); + case value_t::null: + return BasicJsonType(nullptr); + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + BasicJsonType result; sax_t sax(result, true); const string_t no_token{}; - std::vector open((std::max)(d.max_depth, std::size_t{1})); + // (shallow subtrees on the stack: a call for a small value then allocates only for the value) + std::array shallow; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled as containers open + std::vector deep{}; + frame* open = shallow.data(); + std::size_t capacity = shallow.size(); + if (d.max_depth > capacity) + { + deep.resize(d.max_depth); + open = deep.data(); + capacity = deep.size(); + } std::size_t depth = 0; for (;;) { @@ -76,9 +110,13 @@ BasicJsonType materialize(const document_data& d, const node* n) { sax.start_array(n->len); } - if (NLOHMANN_VIEW_UNLIKELY(depth == open.size())) + if (NLOHMANN_VIEW_UNLIKELY(depth == capacity)) { - open.resize(depth * 2); // (deeper than the recorded nesting, e.g. after edits) + // (deeper than the recorded nesting, e.g. after edits) + deep.assign(open, open + depth); + deep.resize(depth * 2); + open = deep.data(); + capacity = deep.size(); } open[depth++] = frame{nav::first(d, n), nav::end(d, n), object}; break; @@ -105,6 +143,8 @@ BasicJsonType materialize(const document_data& d, const node* n) case value_t::binary: case value_t::discarded: default: + // false positive: the parser's frame stack points into its own storage (never null) + // @infer-ignore NULLPTR_DEREFERENCE sax.null(); break; } diff --git a/include/nlohmann/detail/view/object_index.hpp b/include/nlohmann/detail/view/object_index.hpp index 5c7551fe9..9b0ba63ea 100644 --- a/include/nlohmann/detail/view/object_index.hpp +++ b/include/nlohmann/detail/view/object_index.hpp @@ -47,27 +47,96 @@ constexpr std::size_t index_max_displacement = 64; static_assert(document_data::index_min_members >= 2 * index_max_displacement, "a table must be much larger than the probe window of a lookup"); -/// hash of a key: its bytes, eight at a time, in a fixed byte order +/// hash of a key: its bytes, eight at a time (in a fixed byte order), where the +/// last word may overlap the one before (so that no loop over single bytes is +/// needed); a key of fewer than eight bytes is read as two overlapping words of +/// four or two bytes (in the native byte order: the hash is not stored, and the +/// same on both sides of a lookup) inline std::uint64_t key_hash(const char* s, std::size_t n) noexcept { std::uint64_t h = 0x9E3779B97F4A7C15u * (n + 1); const auto* p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - while (n >= 8) - { - h = (h ^ read_eight_bytes(p)) * 0xBF58476D1CE4E5B9u; - h ^= h >> 29u; - p += 8; - n -= 8; - } std::uint64_t w = 0; - for (std::size_t i = 0; i < n; ++i) + if (n >= 8) { - w |= static_cast(p[i]) << (8u * i); + const unsigned char* const last = p + n - 8; + while (p < last) + { + h = (h ^ read_eight_bytes(p)) * 0xBF58476D1CE4E5B9u; + h ^= h >> 29u; + p += 8; + } + w = read_eight_bytes(last); } - h = (h ^ w) * 0x94D049BB133111EBu; + else if (n >= 4) + { + std::uint32_t head = 0; + std::uint32_t tail = 0; + std::memcpy(&head, p, 4); + std::memcpy(&tail, p + n - 4, 4); + w = static_cast(head) | (static_cast(tail) << 32u); + } + else if (n >= 2) + { + std::uint16_t head = 0; + std::uint16_t tail = 0; + std::memcpy(&head, p, 2); + std::memcpy(&tail, p + n - 2, 2); + w = static_cast(head) | (static_cast(tail) << 16u); + } + else if (n == 1) + { + w = p[0]; + } + // (a product only carries bits upwards, and the slot is taken from the low + // bits: the high bits of the last word are folded down before the multiplication) + h = (h ^ w ^ (w >> 29u)) * 0x94D049BB133111EBu; return h ^ (h >> 31u); } +/// whether the n <= 16 bytes at a and b are equal, as two overlapping words +/// (the first and the last 8, 4, or 2 bytes) and without a call +NLOHMANN_VIEW_ALWAYS_INLINE bool same_short_bytes(const unsigned char* a, const unsigned char* b, std::size_t n) noexcept +{ + if (n >= 8) + { + std::uint64_t x0 = 0; + std::uint64_t x1 = 0; + std::uint64_t y0 = 0; + std::uint64_t y1 = 0; + std::memcpy(&x0, a, 8); + std::memcpy(&x1, a + n - 8, 8); + std::memcpy(&y0, b, 8); + std::memcpy(&y1, b + n - 8, 8); + return x0 == y0 && x1 == y1; + } + if (n >= 4) + { + std::uint32_t x0 = 0; + std::uint32_t x1 = 0; + std::uint32_t y0 = 0; + std::uint32_t y1 = 0; + std::memcpy(&x0, a, 4); + std::memcpy(&x1, a + n - 4, 4); + std::memcpy(&y0, b, 4); + std::memcpy(&y1, b + n - 4, 4); + return x0 == y0 && x1 == y1; + } + if (n >= 2) + { + std::uint16_t x0 = 0; + std::uint16_t x1 = 0; + std::uint16_t y0 = 0; + std::uint16_t y1 = 0; + std::memcpy(&x0, a, 2); + std::memcpy(&x1, a + n - 2, 2); + std::memcpy(&y0, b, 2); + std::memcpy(&y1, b + n - 2, 2); + return x0 == y0 && x1 == y1; + } + return n == 0 || *a == *b; +} + /// build the table of a large object inline void build_object_index(document_data& d, node* obj) { @@ -132,6 +201,27 @@ inline const node* find_indexed(const document_data& d, const node* obj, const c const document_data::object_index& ix = d.indexes[obj->extra - 1u]; const std::uint32_t* const table = d.index_slots.data() + ix.start; std::size_t slot = conditional_static_cast(key_hash(key, n)) & ix.mask; + const auto* const k_bytes = reinterpret_cast(key); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + // (the loop for longer keys comes first, as the code is laid out; it is not the more common case) + if (NLOHMANN_VIEW_UNLIKELY(n <= 16)) + { + // (up to 16 bytes, the bytes are compared as two words, without a call) + for (std::size_t distance = 0; distance <= index_max_displacement; ++distance) + { + const std::uint32_t s = table[slot]; + if (s == 0) + { + return nullptr; + } + const node* const k = obj + s; + if (k->len == n && same_short_bytes(reinterpret_cast(d.str(*k)), k_bytes, n)) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + { + return k; + } + slot = (slot + 1) & ix.mask; + } + return nullptr; + } for (std::size_t distance = 0; distance <= index_max_displacement; ++distance) { const std::uint32_t s = table[slot]; diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 7d393d0f6..dbaca6171 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -23,6 +23,7 @@ #include #include #include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -80,13 +81,6 @@ class output_buffer put(s, N - 1); } - void put_repeated(char c, std::size_t n) - { - reserve(n); - std::memset(m_pos, c, n); - m_pos += n; - } - /// the write position and the end of the writable space, for a writer /// that keeps the position in a local variable (set_cursor() hands it back) char* cursor() const noexcept @@ -143,9 +137,27 @@ class output_buffer /// image_check::bounds can have others, which are then written unchanged). inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexcept { + std::size_t i = 0; +#if NLOHMANN_VIEW_SSE2 + // (16 bytes per step; with NEON, the mask of a step costs more than it saves on the + // short runs that are typical here) + // NOLINTBEGIN(portability-simd-intrinsics) + for (; i + 16 <= n; i += 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(s + i))); // NOLINT(bugprone-casting-through-void) + // (a byte is below 0x20 if it is its own minimum with 0x1F) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmpeq_epi8(_mm_min_epu8(in, _mm_set1_epi8(0x1F)), in)); + const auto mask = static_cast(static_cast(_mm_movemask_epi8(special))); + if (mask != 0) + { + return i + static_cast(count_trailing_zeros(mask)); + } + } + // NOLINTEND(portability-simd-intrinsics) +#endif constexpr std::uint64_t ones = 0x0101010101010101ull; constexpr std::uint64_t high = 0x8080808080808080ull; - std::size_t i = 0; for (; i + 8 <= n; i += 8) { const std::uint64_t v = read_eight_bytes(s + i); @@ -196,6 +208,70 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept return true; } +#if NLOHMANN_VIEW_VECTOR +/// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a +/// backslash, a control character, DEL, or not ASCII; a signed compare with +/// 0x20 finds control characters and non-ASCII bytes at once), one bit per +/// byte (x86) or one nibble per byte (NEON) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t in = vld1q_u8(p); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)))); + return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20)))); + return static_cast(static_cast(_mm_movemask_epi8(special))); +#endif +} + +/// the width in bits of a byte in the mask of ascii_stop_mask() +constexpr unsigned ascii_stop_bits() noexcept +{ + return NLOHMANN_VIEW_NEON ? 4u : 1u; +} +#endif + +/// As find_ascii_copyable_run(), 16 bytes per step. end is the end of the +/// buffer that holds the n bytes at p (a short run is read as 16 bytes if +/// they are in the buffer). +inline std::size_t ascii_copyable_run(const unsigned char* p, std::size_t n, const unsigned char* end) noexcept +{ +#if NLOHMANN_VIEW_VECTOR + if (n >= 16) + { + std::size_t i = 0; + for (; i + 16 <= n; i += 16) + { + const std::uint64_t mask = ascii_stop_mask(p + i); + if (mask != 0) + { + return i + (static_cast(count_trailing_zeros(mask)) / ascii_stop_bits()); + } + } + if (i == n) + { + return n; + } + // the last 16 bytes: those before i are known to be copyable + const std::uint64_t mask = ascii_stop_mask(p + n - 16); + return mask != 0 ? n - 16 + (static_cast(count_trailing_zeros(mask)) / ascii_stop_bits()) : n; + } + if (end - p >= 16) + { + // (the bytes after the run are masked out) + const std::uint64_t mask = ascii_stop_mask(p) & ((std::uint64_t{1} << (n * ascii_stop_bits())) - 1u); + return mask != 0 ? static_cast(count_trailing_zeros(mask)) / ascii_stop_bits() : n; + } +#else + static_cast(end); +#endif + return find_ascii_copyable_run(p, n); +} + /// A stack that starts in a buffer of the caller (a local array) and moves to /// the heap (a vector of the caller) only when that is full, so that dumps of /// shallow documents need no allocation. The top is a pointer, as in @@ -275,6 +351,39 @@ NLOHMANN_VIEW_ALWAYS_INLINE char* put_literal(char* w, const char (&s)[N]) noexc return w + N - 1; } +/// the depth of the innermost open container, counted only for indented output +template +struct depth_counter +{ + std::size_t value = 0; + void enter() noexcept + { + ++value; + } + std::size_t leave() noexcept + { + return --value; + } + std::size_t get() const noexcept + { + return value; + } +}; + +template<> +struct depth_counter +{ + void enter() noexcept {} + std::size_t leave() noexcept // NOLINT(readability-convert-member-functions-to-static): same interface as depth_counter + { + return 0; + } + std::size_t get() const noexcept // NOLINT(readability-convert-member-functions-to-static): same interface as depth_counter + { + return 0; + } +}; + /// how the view's dump() writes a value struct dump_style { @@ -311,139 +420,95 @@ class view_serializer void dump(const node* root) { - if (!m_style.pretty) + if (m_style.ensure_ascii) { - if (m_style.ensure_ascii) - { - dump_compact_ascii(root); - } - else if (m_style.source_numbers) - { - dump_compact(root); - } - else - { - dump_compact(root); - } - return; + dump_ascii(root); } - struct frame + else if (m_style.pretty) { - const node* pos; ///< next element, or key of the next member - const node* end; - bool object; - bool first; ///< nothing written yet - }; - std::array buffer; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read - std::vector heap; - small_stack stack(buffer.data(), buffer.size(), heap); - const node* n = root; - for (;;) + dump_pretty(root); + } + else if (m_style.source_numbers) { - // write the value at n - if (is_container(*n)) - { - const bool object = n->kind == static_cast(value_t::object); - if (n->len == 0) - { - m_out.put(object ? "{}" : "[]", 2); - } - else - { - m_out.put(object ? '{' : '['); - stack.push_back(frame{nav::first(m_doc, n), nav::end(m_doc, n), object, true}); - } - } - else - { - write_scalar(*n); - } - - // go to the next value: close finished containers, then separate - for (;;) - { - if (stack.empty()) - { - m_out.finish(); - return; - } - if (stack.back().pos == stack.back().end) - { - const bool object = stack.back().object; - stack.pop_back(); - newline(stack.size()); - m_out.put(object ? '}' : ']'); - continue; - } - frame& f = stack.back(); - if (!f.first) - { - m_out.put(','); - } - f.first = false; - newline(stack.size()); - if (f.object) - { - write_string(*f.pos); - if (m_style.pretty) - { - m_out.put(": "); - } - else - { - m_out.put(':'); - } - n = nav::value(f.pos + 1); - f.pos = document_data::after(f.pos + 1); - } - else - { - n = nav::value(f.pos); - f.pos = document_data::after(f.pos); - } - break; - } + dump_text(root); + } + else + { + dump_text(root); } } private: + /// dump_text() with ensure_ascii, out of line: the walk without it is + /// faster when it is not merged with this one into dump() + NLOHMANN_VIEW_NOINLINE void dump_ascii(const node* root) + { + if (m_style.pretty) + { + if (m_style.source_numbers) + { + dump_text(root); + } + else + { + dump_text(root); + } + } + else if (m_style.source_numbers) + { + dump_text(root); + } + else + { + dump_text(root); + } + } + + /// dump_text() with indentation, out of line: the compact walk is faster + /// when it is not merged with the indented ones into dump() + NLOHMANN_VIEW_NOINLINE void dump_pretty(const node* root) + { + if (m_style.source_numbers) + { + dump_text(root); + } + else + { + dump_text(root); + } + } + /// a string that is not a plain string of the source (decoded, or written /// by an edit), without ensure_ascii: runs without characters to escape - /// are copied - NLOHMANN_VIEW_NOINLINE void write_decoded(const node& n) + /// are copied. If validate is set, the string must be valid UTF-8: the + /// runs are found by the lexer's scan, which validates the sequences, and + /// invalid UTF-8 throws what basic_json::dump() throws. (The compact output + /// writes the bytes of a damaged image unchanged, and scans without + /// validating them.) + NLOHMANN_VIEW_NOINLINE void write_decoded(const node& n, bool validate) { const auto* const s = reinterpret_cast(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) m_out.put('"'); for (std::size_t i = 0; i < n.len;) { - const std::size_t run = plain_output_run(s + i, n.len - i); + const std::size_t run = validate ? string_bulk_run(s + i, n.len - i) : plain_output_run(s + i, n.len - i); if (run != 0) { m_out.put(reinterpret_cast(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) i += run; continue; } + if (validate && s[i] >= 0x80) + { + invalid_utf8(s, n.len); // (the scan stopped at ill-formed UTF-8) + } write_codepoint(s[i], s + i, 1); // a quote, a backslash, or a control character ++i; } m_out.put('"'); } - /// dump_compact() with ensure_ascii, out of line: the walk without it is - /// faster when it is not merged with this one into dump() - NLOHMANN_VIEW_NOINLINE void dump_compact_ascii(const node* root) - { - if (m_style.source_numbers) - { - dump_compact(root); - } - else - { - dump_compact(root); - } - } - - /// whether dump_compact() copies a string or key from the source: one of + /// whether dump_text() copies a string or key from the source: one of /// the source, and with ensure_ascii one without bytes >= 0x7F (an overload /// each, so that the walk without ensure_ascii is compiled exactly as /// before: with GCC, a condition that only folds away later changed its @@ -458,13 +523,155 @@ class view_serializer return (n.flags & node_flags::storage) == 0 && is_ascii_without_delete(src + n.off, n.len); } - /// a string with ensure_ascii (dump_compact()), out of line - NLOHMANN_VIEW_NOINLINE void write_ascii_string(const node& n) + /*! + @brief a string with ensure_ascii: the characters that are not printable ASCII as \\uXXXX + (a character beyond U+FFFF as a surrogate pair); strings of a document + are valid UTF-8, except in a damaged image loaded with image_check::bounds, + for which this throws what basic_json::dump() throws for the string + */ + NLOHMANN_VIEW_NOINLINE void write_ascii(const node& n) { - write_string(n); + const auto* const s = reinterpret_cast(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const std::size_t size = n.len; + char* w = m_out.cursor(); + char* lim = m_out.limit(); + // room for k bytes + const auto room = [&](std::size_t k) + { + const auto left = static_cast(lim - w); + if (NLOHMANN_VIEW_UNLIKELY(left < k)) + { + m_out.set_cursor(w); + m_out.reserve(k); + w = m_out.cursor(); + lim = m_out.limit(); + } + }; + static constexpr const char* hex = "0123456789abcdef"; + // \\uXXXX + const auto u_escape = [&](std::uint32_t u) + { + w[0] = '\\'; + w[1] = 'u'; + w[2] = hex[(u >> 12u) & 0xFu]; + w[3] = hex[(u >> 8u) & 0xFu]; + w[4] = hex[(u >> 4u) & 0xFu]; + w[5] = hex[u & 0xFu]; + w += 6; + }; + room(1); + *w++ = '"'; + std::size_t i = 0; + // (a string of the source starts with a run that has nothing to escape, 16 bytes per step) + const std::size_t prefix = (n.flags & node_flags::storage) == 0 ? ascii_copyable_run(s, size, reinterpret_cast(m_doc.src + m_doc.size)) : 0; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + if (prefix != 0) + { + room(prefix); + std::memcpy(w, s, prefix); + w += prefix; + i = prefix; + } + while (i < size) + { + // (a character that needs an escape sequence is not worth a call) + const std::size_t run = is_ascii_copyable(s[i]) ? find_ascii_copyable_run(s + i, size - i) : 0; + if (run != 0) + { + room(run); + std::memcpy(w, s + i, run); + w += run; + i += run; + continue; + } + room(12); + if (s[i] >= 0x80) + { + const unsigned char lead = s[i]; + // (the two-byte sequences of Latin text and the three-byte sequences of most other + // text, with continuation bytes of any value: the ranges that are restricted (E0, + // ED, and the sequences of four bytes) go through validate_one_utf8()) + if (lead >= 0xE1 && lead <= 0xEF && lead != 0xED && size - i >= 3 && (s[i + 1] & 0xC0u) == 0x80 && (s[i + 2] & 0xC0u) == 0x80) + { + u_escape(((lead & 0x0Fu) << 12u) | ((s[i + 1] & 0x3Fu) << 6u) | (s[i + 2] & 0x3Fu)); + i += 3; + continue; + } + if (lead >= 0xC2 && lead <= 0xDF && size - i >= 2 && (s[i + 1] & 0xC0u) == 0x80) + { + u_escape(((lead & 0x1Fu) << 6u) | (s[i + 1] & 0x3Fu)); + i += 2; + continue; + } + const std::size_t len = validate_one_utf8(s + i, size - i); + if (NLOHMANN_VIEW_UNLIKELY(len == 0)) + { + m_out.set_cursor(w); + invalid_utf8(s, size); + return; // LCOV_EXCL_LINE (invalid_utf8 throws) + } + std::uint32_t codepoint = s[i] & (0xFFu >> (len + 1)); + for (std::size_t k = 1; k < len; ++k) + { + codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu); + } + i += len; + if (codepoint <= 0xFFFF) + { + u_escape(codepoint); + } + else + { + u_escape(0xD7C0u + (codepoint >> 10u)); + u_escape(0xDC00u + (codepoint & 0x3FFu)); + } + continue; + } + // a quote, a backslash, or a control character (or DEL) + const unsigned char c = s[i++]; + char escape = 0; + switch (c) + { + case 0x08: + escape = 'b'; + break; + case 0x09: + escape = 't'; + break; + case 0x0A: + escape = 'n'; + break; + case 0x0C: + escape = 'f'; + break; + case 0x0D: + escape = 'r'; + break; + case 0x22: + escape = '"'; + break; + case 0x5C: + escape = '\\'; + break; + default: + break; + } + if (escape != 0) + { + w[0] = '\\'; + w[1] = escape; + w += 2; + } + else + { + u_escape(c); + } + } + room(1); + *w++ = '"'; + m_out.set_cursor(w); } - /// the copies of dump_compact() that are not fixed-size moves (long + /// the copies of dump_text() that are not fixed-size moves (long /// strings, or near the end of the source); out of line, so that the /// compiler does not merge the fixed-size moves into this call NLOHMANN_VIEW_NOINLINE static void copy_long(char* to, const char* from, std::size_t n) noexcept @@ -481,7 +688,7 @@ class view_serializer } /*! - @brief the compact output (the default dump(), or with ensure_ascii) + @brief the output, compact (the default dump()) or indented, with or without ensure_ascii The same walk as dump(), with the write position in a local variable (stores through char pointers would otherwise force a reload of the @@ -489,11 +696,11 @@ class view_serializer the source copied by fixed-size moves of 32 bytes where the source has that many bytes left, instead of a library call per token. The buffer keeps 64 bytes of slack for the overshoot. With ensure_ascii (Ascii), a - string of the source is copied only if it has no byte >= 0x7F (see - is_ascii_without_delete()); any other string is written by write_string(). + string of the source is copied only if it has nothing to escape (see + copied()); any other string is written by write_ascii(). */ - template - void dump_compact(const node* root) + template + void dump_text(const node* root) { struct frame { @@ -508,6 +715,16 @@ class view_serializer const char* const src_end = src + m_doc.size; char* w = m_out.cursor(); char* lim = m_out.limit(); + // (indented output) a line break and 63 indentation characters, from + // which the line break of a level of up to 63 characters is a copy + std::array ruler; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled below if used + if (Pretty) + { + ruler.fill(m_style.indent_char); + ruler[0] = '\n'; + } + const std::size_t indent = m_style.indent; + depth_counter level; // depth of the innermost open container (indented output) // room for n bytes and the slack const auto room = [&](std::size_t n) { @@ -539,6 +756,22 @@ class view_serializer } w += n; }; + // (indented output) the line break and the indentation of a level + const auto newline = [&](std::size_t depth) + { + const std::size_t count = 1 + (depth * indent); + room(count); + if (count <= ruler.size()) + { + std::memcpy(w, ruler.data(), ruler.size()); + } + else + { + *w = '\n'; + std::memset(w + 1, m_style.indent_char, count - 1); + } + w += count; + }; // a string that is not a plain string of the source (out of line, so // that the cursor stays in a register here) const auto escaped = [&](const node & n) @@ -546,11 +779,11 @@ class view_serializer m_out.set_cursor(w); if (enabled(Ascii)) { - write_ascii_string(n); + write_ascii(n); } else { - write_decoded(n); + write_decoded(n, Pretty); } w = m_out.cursor(); lim = m_out.limit(); @@ -651,6 +884,7 @@ class view_serializer { *w++ = object ? '{' : '['; stack.push_back(cur); + level.enter(); if (Editable) { cur = frame{nav::first(m_doc, n), nav::end(m_doc, n), object}; @@ -693,6 +927,10 @@ class view_serializer { break; } + if (Pretty) + { + newline(level.leave()); + } room(1); *w++ = cur.object ? '}' : ']'; cur = stack.back(); @@ -701,13 +939,17 @@ class view_serializer room(1); *w++ = ','; } + if (Pretty) + { + newline(level.get()); + } const node* const at = Editable ? cur.pos : n; if (cur.object) { const node& key = *at; if (copied(key, src, std::integral_constant())) { - room(key.len + 3); + room(key.len + (Pretty ? 4 : 3)); *w++ = '"'; copy(src + key.off, key.len); w[0] = '"'; @@ -717,9 +959,13 @@ class view_serializer else { escaped(key); - room(1); + room(Pretty ? 2 : 1); *w++ = ':'; } + if (Pretty) + { + *w++ = ' '; + } if (Editable) { n = nav::value(at + 1); @@ -738,69 +984,6 @@ class view_serializer } } - void newline(std::size_t level) - { - if (m_style.pretty) - { - m_out.put('\n'); - m_out.put_repeated(m_style.indent_char, level * m_style.indent); - } - } - - void write_scalar(const node& n) - { - switch (static_cast(n.kind)) - { - case value_t::null: - m_out.put("null"); - break; - case value_t::boolean: - if ((n.flags & node_flags::is_true) != 0) - { - m_out.put("true"); - } - else - { - m_out.put("false"); - } - break; - case value_t::string: - write_string(n); - break; - case value_t::number_integer: - case value_t::number_unsigned: - { - const char* const token = m_doc.str(n); - const std::uint32_t len = number_length(n); - if (!m_style.source_numbers && len == 2 && token[0] == '-' && token[1] == '0') - { - m_out.put('0'); // parse() reads -0 as the integer 0 - } - else - { - m_out.put(token, len); - } - break; - } - case value_t::number_float: - if (m_style.source_numbers && (n.flags & node_flags::storage) != node_flags::edited) - { - m_out.put(m_doc.str(n), n.len); // (a float set by an edit is written as with shortest) - } - else - { - write_float_node(n); - } - break; - case value_t::object: // LCOV_EXCL_LINE (containers are written by dump()) - case value_t::array: // LCOV_EXCL_LINE - case value_t::binary: // LCOV_EXCL_LINE (not in a document) - case value_t::discarded: // LCOV_EXCL_LINE - default: // LCOV_EXCL_LINE - break; // LCOV_EXCL_LINE - } - } - /// a float node as dump() writes it void write_float_node(const node& n) { @@ -911,72 +1094,6 @@ class view_serializer m_out.put(s.data(), s.size()); } - void write_string(const node& n) - { - const char* const s = m_doc.str(n); - m_out.put('"'); - if ((n.flags & node_flags::storage) == 0 && (!m_style.ensure_ascii || is_ascii_without_delete(s, n.len))) - { - // a string of the source without escape sequences has nothing to - // escape (with ensure_ascii, if it is printable ASCII) - m_out.put(s, n.len); - } - else if (m_style.ensure_ascii) - { - write_escaped(reinterpret_cast(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - else - { - write_escaped(reinterpret_cast(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - m_out.put('"'); - } - - /// as serializer::dump_escaped(); strings of a document are valid UTF-8, - /// except in a damaged image loaded with image_check::bounds, for which - /// this throws what basic_json::dump() throws for the string - template - void write_escaped(const unsigned char* s, std::size_t n) - { - std::size_t i = 0; - while (i < n) - { - std::size_t run = 0; - if (!EnsureAscii) - { - run = string_bulk_run(s + i, n - i); - } - else if (is_ascii_copyable(s[i])) - { - run = find_ascii_copyable_run(s + i, n - i); - } - if (run != 0) - { - m_out.put(reinterpret_cast(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - i += run; - continue; - } - std::uint32_t codepoint = s[i]; - std::size_t len = 1; - if (codepoint >= 0x80) - { - len = validate_one_utf8(s + i, n - i); - if (NLOHMANN_VIEW_UNLIKELY(len == 0)) - { - invalid_utf8(s, n); - return; - } - codepoint &= 0xFFu >> (len + 1); - for (std::size_t k = 1; k < len; ++k) - { - codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu); - } - } - write_codepoint(codepoint, s + i, len); - i += len; - } - } - /// throw what basic_json::dump() throws for a string that is not valid UTF-8 NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n) { diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index b83cd5a3b..7a0248f63 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -116,6 +116,34 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned } return p; } + +/*! +@brief copy 16 bytes from p to w, and find the first quote, backslash, +control character, or non-ASCII byte among them + +The bytes are copied whether or not they are all plain (the caller has room +for 16 bytes and keeps only the plain ones). + +@return the offset of that byte, or 16 if there is none +*/ +NLOHMANN_VIEW_ALWAYS_INLINE unsigned copy_plain_run16(const unsigned char* p, char* w) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t in = vld1q_u8(p); + vst1q_u8(reinterpret_cast(w), in); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); + return bits != 0 ? static_cast(count_trailing_zeros(bits) >> 2u) : 16u; +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + _mm_storeu_si128(static_cast<__m128i*>(static_cast(w)), in); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); + return bits != 0 ? static_cast(count_trailing_zeros(bits)) : 16u; +#endif +} #endif #if NLOHMANN_VIEW_SSSE3_DISPATCH diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 9c0e11632..ff501460a 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -19,6 +19,7 @@ #define INCLUDE_NLOHMANN_JSON_HPP_ #include // all_of, find, for_each, none_of +#include // array #include // isnan #include // nullptr_t, ptrdiff_t, size_t #include // uint8_t @@ -158,9 +159,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend class ::nlohmann::detail::binary_writer; template friend class ::nlohmann::detail::binary_reader; - template + template friend class ::nlohmann::detail::json_sax_dom_parser; template + friend class ::nlohmann::detail::json_sax_dom_binary_parser; + template friend class ::nlohmann::detail::json_sax_dom_callback_parser; #if JSON_DIAGNOSTIC_POSITIONS friend struct ::nlohmann::detail::diagnostic_positions; @@ -6287,7 +6290,7 @@ public: from_ubjson/from_bjdata/from_bon8/from_bson overloads Building the @ref binary_reader as a named local, rather than as a - temporary that @ref detail::json_sax_dom_parser::parse is called on in + temporary that @ref detail::json_sax_dom_binary_parser::parse is called on in the same expression, avoids a false-positive cppcheck accessMoved warning in each of the 16 callers. */ @@ -6298,7 +6301,7 @@ public: const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { basic_json result; - detail::json_sax_dom_parser sdp(result, allow_exceptions); + detail::json_sax_dom_binary_parser sdp(result, allow_exceptions); binary_reader reader(std::move(ia), format, error_handler); if (!reader.sax_parse(&sdp, strict, tag_handler)) { @@ -6321,7 +6324,7 @@ public: const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_cbor(j); return result; } @@ -6331,7 +6334,9 @@ public: static void to_cbor(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_cbor(j); + binary_writer writer(o, error_handler); + writer.write_cbor(j); + writer.flush(); } /// @brief create a CBOR serialization of a given JSON value @@ -6339,7 +6344,9 @@ public: static void to_cbor(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_cbor(j); + binary_writer writer(o, error_handler); + writer.write_cbor(j); + writer.flush(); } /// @brief create a MessagePack serialization of a given JSON value @@ -6348,7 +6355,7 @@ public: const error_handler_t error_handler = error_handler_t::keep) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_msgpack(j); return result; } @@ -6358,7 +6365,9 @@ public: static void to_msgpack(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = error_handler_t::keep) { - binary_writer(o, error_handler).write_msgpack(j); + binary_writer writer(o, error_handler); + writer.write_msgpack(j); + writer.flush(); } /// @brief create a MessagePack serialization of a given JSON value @@ -6366,7 +6375,9 @@ public: static void to_msgpack(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = error_handler_t::keep) { - binary_writer(o, error_handler).write_msgpack(j); + binary_writer writer(o, error_handler); + writer.write_msgpack(j); + writer.flush(); } /// @brief create a UBJSON serialization of a given JSON value @@ -6377,7 +6388,7 @@ public: const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_ubjson(j, use_size, use_type); return result; } @@ -6388,7 +6399,9 @@ public: const bool use_size = false, const bool use_type = false, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_ubjson(j, use_size, use_type); + binary_writer writer(o, error_handler); + writer.write_ubjson(j, use_size, use_type); + writer.flush(); } /// @brief create a UBJSON serialization of a given JSON value @@ -6397,7 +6410,9 @@ public: const bool use_size = false, const bool use_type = false, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_ubjson(j, use_size, use_type); + binary_writer writer(o, error_handler); + writer.write_ubjson(j, use_size, use_type); + writer.flush(); } /// @brief create a BJData serialization of a given JSON value @@ -6409,7 +6424,7 @@ public: const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -6421,7 +6436,9 @@ public: const bjdata_version_t version = bjdata_version_t::draft2, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_ubjson(j, use_size, use_type, true, true, version); + binary_writer writer(o, error_handler); + writer.write_ubjson(j, use_size, use_type, true, true, version); + writer.flush(); } /// @brief create a BJData serialization of a given JSON value @@ -6431,7 +6448,9 @@ public: const bjdata_version_t version = bjdata_version_t::draft2, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_ubjson(j, use_size, use_type, true, true, version); + binary_writer writer(o, error_handler); + writer.write_ubjson(j, use_size, use_type, true, true, version); + writer.flush(); } /// @brief create a BSON serialization of a given JSON value @@ -6440,7 +6459,7 @@ public: const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_bson(j); return result; } @@ -6450,7 +6469,9 @@ public: static void to_bson(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_bson(j); + binary_writer writer(o, error_handler); + writer.write_bson(j); + writer.flush(); } /// @brief create a BSON serialization of a given JSON value @@ -6458,7 +6479,9 @@ public: static void to_bson(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_bson(j); + binary_writer writer(o, error_handler); + writer.write_bson(j); + writer.flush(); } /// @brief create a BON8 serialization of a given JSON value @@ -6466,7 +6489,7 @@ public: static std::vector to_bon8(const basic_json& j) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result).write_bon8(j); return result; } @@ -6475,14 +6498,18 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ static void to_bon8(const basic_json& j, detail::output_adapter o) { - binary_writer(o).write_bon8(j); + binary_writer writer(o); + writer.write_bon8(j); + writer.flush(); } /// @brief create a BON8 serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ static void to_bon8(const basic_json& j, detail::output_adapter o) { - binary_writer(o).write_bon8(j); + binary_writer writer(o); + writer.write_bon8(j); + writer.flush(); } /// @brief create a JSON value from an input in CBOR format @@ -6878,6 +6905,72 @@ public: /// @name JSON Patch functions /// @{ + private: + /// the members of an operation of a JSON patch that are of interest + struct patch_members + { + basic_json* op = nullptr; + basic_json* path = nullptr; + basic_json* value = nullptr; + basic_json* from = nullptr; + }; + + /// @brief find the members "op", "path", "value", and "from" of the object @a val in one pass + static patch_members find_patch_members(const basic_json& val, std::true_type /*key_type is string_t*/) + { + patch_members members; + for (auto& member : *val.m_data.m_value.object) + { + const string_t& key = member.first; + switch (key.size()) + { + case 2: + if (key == "op") + { + members.op = &member.second; + } + break; + case 4: + if (key == "path") + { + members.path = &member.second; + } + else if (key == "from") + { + members.from = &member.second; + } + break; + case 5: + if (key == "value") + { + members.value = &member.second; + } + break; + default: + break; + } + } + return members; + } + + /// @brief find the members by lookup for key types other than string_t + static patch_members find_patch_members(const basic_json& val, std::false_type /*key_type is not string_t*/) + { + const auto find_member = [&val](const char* name) -> basic_json * + { + const auto it = val.m_data.m_value.object->find(string_t(name)); + return it != val.m_data.m_value.object->end() ? &it->second : nullptr; + }; + + patch_members members; + members.op = find_member("op"); + members.path = find_member("path"); + members.value = find_member("value"); + members.from = find_member("from"); + return members; + } + + public: /// @brief applies a JSON patch in-place without copying the object /// @sa https://json.nlohmann.me/api/basic_json/patch/ void patch_inplace(const basic_json& json_patch) @@ -7051,18 +7144,18 @@ public: // a diff creates them: apply the operation and return true; return // false, with nothing changed, for all others string_t token; - const string_t value_key = "value"; - const auto try_plain = [&result, &token, &value_key](patch_operations operation, const string_t& path, const basic_json & val) + typename json_pointer::template plain_steps steps; + const auto try_plain = [&result, &token, &steps](patch_operations operation, const string_t& path, const patch_members & members, const basic_json & val) { - if (path.empty() || path.find_first_of('~') != string_t::npos) + if (path.empty()) { return false; } // a missing "value" is reported below - const auto value_it = val.m_data.m_value.object->find(value_key); - const basic_json& value = value_it != val.m_data.m_value.object->end() ? value_it->second : val; - if (operation != patch_operations::remove && value_it == val.m_data.m_value.object->end()) + const basic_json* const value_ptr = members.value; + const basic_json& value = value_ptr != nullptr ? *value_ptr : val; + if (operation != patch_operations::remove && value_ptr == nullptr) { return false; } @@ -7071,7 +7164,7 @@ public: { case patch_operations::replace: { - basic_json* const target = json_pointer::resolve_plain(&result, path, path.size(), token); + basic_json* const target = json_pointer::resolve_plain(&result, path, path.size(), token, steps); if (target == nullptr) { return false; @@ -7082,7 +7175,7 @@ public: case patch_operations::test: { - const basic_json* const target = json_pointer::resolve_plain(&result, path, path.size(), token); + const basic_json* const target = json_pointer::resolve_plain(&result, path, path.size(), token, steps); // a failing test is reported below return target != nullptr && *target == value; } @@ -7102,13 +7195,17 @@ public: } --slash; - basic_json* const parent = json_pointer::resolve_plain(&result, path, slash, token); + basic_json* const parent = json_pointer::resolve_plain(&result, path, slash, token, steps); if (parent == nullptr) { return false; } token.clear(); token.append(path.data() + slash + 1, path.size() - slash - 1); + if (token.find_first_of('~') != string_t::npos) + { + return false; + } if (operation == patch_operations::add) { @@ -7164,31 +7261,32 @@ public: } }; + // the name of the first member to check + const string_t op_member = "op"; + // iterate and apply the operations for (const auto& val : json_patch) { // wrapper to get a value for an operation - const auto get_value = [&val](const string_t& op, - const string_t& member, + const auto get_value = [&val](basic_json * found, + const string_t& op, + const char* member, bool string_type) -> basic_json & { - // find value - auto it = val.m_data.m_value.object->find(member); - // check if the desired value is present - if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end())) + if (JSON_HEDLEY_UNLIKELY(found == nullptr)) { - detail::throw_patch_missing_member(op, member, false, val); + detail::throw_patch_missing_member(op, string_t(member), false, val); } // check if the result is of type string - if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) + if (JSON_HEDLEY_UNLIKELY(string_type && !found->is_string())) { - detail::throw_patch_missing_member(op, member, true, val); + detail::throw_patch_missing_member(op, string_t(member), true, val); } // no error: return value - return it->second; + return *found; }; // type check: every element of the array must be an object @@ -7197,26 +7295,30 @@ public: detail::throw_patch_not_an_array(val); } + const patch_members members = find_patch_members(val, std::is_same {}); + // collect mandatory members - const string_t& op = *get_value("op", "op", true).m_data.m_value.string; - const string_t& path = *get_value(op, "path", true).m_data.m_value.string; + const string_t& op = *get_value(members.op, op_member, "op", true).m_data.m_value.string; + const string_t& path = *get_value(members.path, op, "path", true).m_data.m_value.string; const patch_operations operation = get_op(op); // the usual operations on a plain pointer are applied without // building the pointer's reference tokens; anything else (and // every error) is handled below - if (try_plain(operation, path, val)) + if (try_plain(operation, path, members, val)) { continue; } + // the general operations may change any container + steps.reset(); json_pointer ptr(path); switch (operation) { case patch_operations::add: { - operation_add(ptr, get_value("add", "value", false)); + operation_add(ptr, get_value(members.value, "add", "value", false)); break; } @@ -7229,13 +7331,13 @@ public: case patch_operations::replace: { // the "path" location must exist - use at() - result.at(ptr) = get_value("replace", "value", false); + result.at(ptr) = get_value(members.value, "replace", "value", false); break; } case patch_operations::move: { - const string_t& from_path = *get_value("move", "from", true).m_data.m_value.string; + const string_t& from_path = *get_value(members.from, "move", "from", true).m_data.m_value.string; json_pointer from_ptr(from_path); if (JSON_HEDLEY_UNLIKELY(is_proper_prefix(from_ptr, ptr))) @@ -7258,7 +7360,7 @@ public: case patch_operations::copy: { - const string_t& from_path = *get_value("copy", "from", true).m_data.m_value.string; + const string_t& from_path = *get_value(members.from, "copy", "from", true).m_data.m_value.string; const json_pointer from_ptr(from_path); // The copy is functionally identical to an "add" @@ -7276,7 +7378,7 @@ public: { // check if "value" matches the one at "path" // the "path" location must exist - use at() - success = (result.at(ptr) == get_value("test", "value", false)); + success = (result.at(ptr) == get_value(members.value, "test", "value", false)); } JSON_INTERNAL_CATCH (out_of_range&) { @@ -7368,13 +7470,18 @@ public: /// @brief append an operation @a op for @a path to @a result, with @a value if it is not null static void diff_operation(basic_json& result, const char* op, const string_t& path, const basic_json* value) { + // the keys are distinct and in the order of std::map; the members are + // added without looking them up basic_json entry(value_t::object); - entry["op"] = op; - entry["path"] = path; + auto& members = *entry.m_data.m_value.object; + auto hint = members.end(); + json_pointer::add_flattened(members, hint, string_t("op"), string_t(op), detail::priority_tag<2> {}); + json_pointer::add_flattened(members, hint, string_t("path"), path, detail::priority_tag<2> {}); if (value != nullptr) { - entry["value"] = *value; + json_pointer::add_flattened(members, hint, string_t("value"), *value, detail::priority_tag<2> {}); } + entry.set_parents(); result.push_back(std::move(entry)); } @@ -7396,6 +7503,62 @@ public: diff_operation(result, "add", path, &value); } + /// @brief the values in the target of the members of a source object, for the + /// objects of up to 16 members without allocating + /// + /// reserve() must be called with the number of values that are added. + class diff_target_values + { + public: + diff_target_values() noexcept + : inline_items() // NOLINT(readability-redundant-member-init) + {} + diff_target_values(const diff_target_values&) = delete; + diff_target_values& operator=(const diff_target_values&) = delete; + diff_target_values(diff_target_values&&) = delete; + diff_target_values& operator=(diff_target_values&&) = delete; + ~diff_target_values() = default; + + void reserve(const std::size_t n) + { + if (n > inline_size) + { + large.resize(n); + items = large.data(); + } + } + + void push_back(const basic_json* value) + { + items[count++] = value; + } + + std::size_t size() const noexcept + { + return count; + } + + const basic_json* operator[](const std::size_t i) const noexcept + { + return items[i]; + } + + private: + static constexpr std::size_t inline_size = 16; + /// the values of a small object, left uninitialized: they are written + /// before they are read (the user-provided constructor keeps the + /// member initialization, which -Weffc++ asks for, from zeroing them) + struct uninitialized_items : std::array + { + uninitialized_items() noexcept {} // NOLINT(modernize-use-equals-default,hicpp-use-equals-default,cppcoreguidelines-pro-type-member-init,hicpp-member-init): see above + }; + + std::size_t count = 0; + std::vector large{}; // NOLINT(readability-redundant-member-init) + uninitialized_items inline_items; + const basic_json** items = inline_items.data(); + }; + /// @brief append "/" and the decimal representation of @a index to @a path static void diff_append_index(string_t& path, std::size_t index) { @@ -7476,8 +7639,9 @@ public: operations that remove every member of @a source and add every member of @a target to @a result, and returns false. */ + template static bool diff_object_keys(basic_json& result, const basic_json& source, const basic_json& target, - const string_t& path, std::vector& target_values, + const string_t& path, TargetValues& target_values, std::vector& added) { // first pass: one lookup per source key @@ -7691,7 +7855,7 @@ public: case value_t::object: { - std::vector target_values; + diff_target_values target_values; std::vector added; if (diff_object_keys(result, source, target, path, target_values, added)) { diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index f0fc58dc9..86481d884 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -30,6 +30,7 @@ #include // array #include // nullptr_t, size_t // IWYU pragma: keep #include // uint8_t, uint32_t +#include // FILE #include // memcpy, strlen #include // istream // IWYU pragma: keep #include // distance, input_iterator_tag, iterator_traits @@ -295,7 +296,7 @@ class basic_json_view } detail::view::throw_type_error(305, "cannot use operator[] with a numeric argument with ", type_name()); } - return idx < m_node->len ? basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))) : basic_json_view(); + return idx < m_node->len ? element(idx) : basic_json_view(); } /// any other integer type (int, unsigned, long, std::int64_t, ...; a @@ -358,7 +359,7 @@ class basic_json_view { detail::view::throw_out_of_range(401, detail::concat("array index ", std::to_string(idx), " is out of range")); } - return basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))); + return element(idx); } /// any other integer type, see operator[]; negative values are out of @@ -392,7 +393,13 @@ class basic_json_view string_t value(string_view_t key, const char* default_value) const { - return value(key, string_t(default_value)); + if (NLOHMANN_VIEW_UNLIKELY(!is_object())) + { + detail::view::throw_type_error(306, "cannot use value() with ", type_name()); + } + // (the default is made only if there is no such member) + const basic_json_view r = lookup(key); + return r.is_discarded() ? string_t(default_value) : r.template get(); } /// the value a JSON pointer refers to converted to T, or the default @@ -411,7 +418,12 @@ class basic_json_view string_t value(const json_pointer& ptr, const char* default_value) const { - return value(ptr, string_t(default_value)); + if (NLOHMANN_VIEW_UNLIKELY(!is_structured())) + { + detail::view::throw_type_error(306, "cannot use value() with ", type_name()); + } + const basic_json_view r = detail::view::resolve_pointer(*this, detail::json_pointer_access::reference_tokens(ptr), detail::view::pointer_mode::value); + return r.is_discarded() ? string_t(default_value) : r.template get(); } /// the first element or member value; a primitive value itself. Throws @@ -433,7 +445,8 @@ class basic_json_view { if (is_structured() && m_node->len != 0) { - return basic_json_view(m_doc, navigation::value(detail::view::last_child(*m_doc, m_node) + (is_object() ? 1 : 0))); + return unedited() ? basic_json_view(m_doc, detail::view::last_child(*m_doc, m_node) + (is_object() ? 1 : 0)) + : basic_json_view(m_doc, navigation::value(detail::view::last_child(*m_doc, m_node) + (is_object() ? 1 : 0))); } return front(); } @@ -450,7 +463,7 @@ class basic_json_view { return end(); } - const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + const node* const k = find_key(key); return k != nullptr ? iterator(m_doc, k, true) : end(); } @@ -467,7 +480,7 @@ class basic_json_view /// whether this is an object with a member with this key bool contains(string_view_t key) const { - return is_object() && detail::view::find_member(*m_doc, m_node, key.data(), key.size()) != nullptr; + return is_object() && find_key(key) != nullptr; } bool contains(const char* key) const @@ -627,7 +640,14 @@ class basic_json_view // the compact writer keeps 64 bytes of slack, so that it does not grow // the buffer just before the end const std::size_t estimate = source_extent() + (style.pretty ? source_extent() / 2 : 0) + 160; - detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); + if (unedited()) + { + detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); + } + else + { + detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); + } return out; } @@ -707,7 +727,7 @@ class basic_json_view { return BasicJsonType(value_t::discarded); } - return detail::view::materialize(*m_doc, m_node); + return unedited() ? detail::view::materialize(*m_doc, m_node) : detail::view::materialize(*m_doc, m_node); } /// byte offset of this value in the source text (for strings: of the @@ -785,10 +805,50 @@ class basic_json_view /// (object required) NLOHMANN_VIEW_ALWAYS_INLINE basic_json_view lookup(string_view_t key) const noexcept { + if (unedited()) + { + const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + return k != nullptr ? basic_json_view(m_doc, k + 1) : basic_json_view(); + } const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); return k != nullptr ? basic_json_view(m_doc, navigation::value(k + 1)) : basic_json_view(); } + /// whether the document has not been edited yet: until the first edit, its index is walked like that of a + /// read-only document (nothing is moved or linked), which is cheaper + NLOHMANN_VIEW_ALWAYS_INLINE bool unedited() const noexcept + { + return !navigation::editable() || m_doc->edits == nullptr; + } + + /// the member of this object with @a key (the last one if the key occurs more than once), or nullptr + NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key) const + { + return find_key(key, std::integral_constant {}); + } + + NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key, std::true_type /*editable*/) const + { + // (an editable document without edits is looked up as one that cannot be edited) + return unedited() ? detail::view::find_member(*m_doc, m_node, key.data(), key.size()) + : detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + } + + NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key, std::false_type /*editable*/) const + { + return detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + } + + /// the element of an array at an index below its size + NLOHMANN_VIEW_ALWAYS_INLINE basic_json_view element(size_type idx) const + { + if (unedited()) + { + return basic_json_view(m_doc, detail::view::element_at(*m_doc, m_node, idx)); + } + return basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))); + } + // --- get() dispatch --- bool get_impl(detail::view::value_tag /*unused*/, detail::priority_tag<2> /*unused*/) const @@ -1528,7 +1588,8 @@ class basic_json_document // NOLINTBEGIN(readability-avoid-nested-conditional-operator): a constant expression of C++11 append_input_impl(buf, std::forward(input), std::integral_constant < int, detail::is_contiguous_byte_container::value ? 0 : std::is_base_of::value ? 1 - : detail::view::is_adapter_input::value ? 2 : 3 > {}); + : detail::view::is_adapter_input::value ? 2 + : detail::view::is_c_stream::value ? 4 : 3 > {}); // NOLINTEND(readability-avoid-nested-conditional-operator) } @@ -1550,6 +1611,19 @@ class basic_json_document detail::view::append_adapter(buf, std::move(input.ia)); } +#ifndef JSON_NO_IO + static void append_input_impl(std::string& buf, std::FILE* file, std::integral_constant /*C stream*/) + { + if (file == nullptr) + { + // (as before: the library's adapter decides what a null stream yields) + detail::view::append_adapter(buf, detail::input_adapter(file)); + return; + } + detail::view::append_file(buf, file); + } +#endif + template static void append_input_impl(std::string& buf, T&& input, std::integral_constant /*other*/) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 89e89c12a..c0a5b82e5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19,6 +19,7 @@ #define INCLUDE_NLOHMANN_JSON_HPP_ #include // all_of, find, for_each, none_of +#include // array #include // isnan #include // nullptr_t, ptrdiff_t, size_t #include // uint8_t @@ -8468,17 +8469,18 @@ NLOHMANN_JSON_NAMESPACE_END -#include // count, find, min +#include // count, find, max, min #include // array #include // size_t #include // uint32_t #include // strlen #include // begin, end, iterator_traits, random_access_iterator_tag, distance, next, reverse_iterator #include // numeric_limits +#include // unique_ptr #include // streambuf #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer -#include // pair, declval +#include // declval, move, pair #include // vector #ifndef JSON_NO_IO @@ -8537,6 +8539,82 @@ enum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata, bon8 }; #ifndef JSON_NO_IO /*! +A growable array of characters for the part of a token that an input adapter +copied out of its window: unlike std::vector::push_back(), appending a +character takes no reference to a local, and is a few instructions when there +is room. +*/ +class token_carry_buffer +{ + public: + token_carry_buffer() = default; + token_carry_buffer(const token_carry_buffer&) = delete; + token_carry_buffer& operator=(const token_carry_buffer&) = delete; + token_carry_buffer(token_carry_buffer&& rhs) noexcept + : m_data(std::move(rhs.m_data)), m_end(rhs.m_end), m_capacity_end(rhs.m_capacity_end) + { + rhs.m_end = rhs.m_capacity_end = nullptr; + } + token_carry_buffer& operator=(token_carry_buffer&&) = delete; + ~token_carry_buffer() = default; + + void clear() noexcept + { + m_end = m_data.get(); + } + + void push_back(char c) + { + if (JSON_HEDLEY_UNLIKELY(m_end == m_capacity_end)) + { + grow(1); + } + *m_end++ = c; + } + + void append(const char* first, const char* last) + { + const auto n = static_cast(last - first); + const auto room = static_cast(m_capacity_end - m_end); + if (n > room) + { + grow(n); + } + if (n != 0) + { + std::memcpy(m_end, first, n); + } + m_end += n; + } + + /// replace the contents of @a out + void copy_to(std::vector& out) const + { + out.assign(m_data.get(), m_end); + } + + private: + /// make room for @a extra more characters + JSON_HEDLEY_NEVER_INLINE void grow(std::size_t extra) + { + const auto size = static_cast(m_end - m_data.get()); + const auto capacity = static_cast(m_capacity_end - m_data.get()); + const std::size_t new_capacity = (std::max)(capacity < 32 ? static_cast(32) : 2 * capacity, size + extra); + std::unique_ptr data(new char[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + if (size != 0) + { + std::memcpy(data.get(), m_data.get(), size); + } + m_data = std::move(data); + m_end = m_data.get() + size; + m_capacity_end = m_data.get() + new_capacity; + } + + std::unique_ptr m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init) + char* m_end = nullptr; + char* m_capacity_end = nullptr; +}; +/*! Input adapter for stdio file access. This adapter read only 1 byte and do not use any buffer. This adapter is a very low level adapter. */ @@ -8674,6 +8752,7 @@ class input_stream_adapter , token_segment(rhs.token_segment) , current_in_get_area(rhs.current_in_get_area) , tracking_token(rhs.tracking_token) + , no_get_area(rhs.no_get_area) #endif { rhs.is = nullptr; @@ -8760,37 +8839,20 @@ class input_stream_adapter std::char_traits::int_type get_character() { #if !JSON_DISABLE_STREAM_BULK_SCAN - // sbumpc() refills the get area if it is exhausted - const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); - if (JSON_HEDLEY_UNLIKELY(refill)) - { - save_token_segment(); - } -#endif - - auto res = sb->sbumpc(); - // set eof manually, as we don't use the istream interface. - if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) - { -#if !JSON_DISABLE_STREAM_BULK_SCAN - current_in_get_area = false; -#endif - is->clear(is->rdstate() | std::ios::eofbit); - } -#if !JSON_DISABLE_STREAM_BULK_SCAN - else if (JSON_HEDLEY_UNLIKELY(refill)) - { - // the consumed character is at gptr() - 1, unless the streambuf - // has no get area and returned it from uflow() - const char* const next = streambuf_access::get_ptr(sb); - after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); - } - else + // the character is in the get area: step over it, as sbumpc() does + const char* const next = streambuf_access::get_ptr(sb); + if (JSON_HEDLEY_LIKELY(next != streambuf_access::get_end(sb))) { current_in_get_area = true; + streambuf_access::bump(sb, 1); + return std::char_traits::to_int_type(*next); + } + if (no_get_area) + { + return get_character_no_get_area(); } #endif - return res; + return get_character_slow(); } #endif @@ -8799,6 +8861,18 @@ class input_stream_adapter { #if JSON_PRECISE_STREAM_POSITION commit_lookahead(); +#endif +#if !JSON_DISABLE_STREAM_BULK_SCAN + // the elements are in the get area: copy them, as sgetn() does, but + // without the virtual call + const char* const first = streambuf_access::get_ptr(sb); + const auto available = static_cast(streambuf_access::get_end(sb) - first); + if (JSON_HEDLEY_LIKELY(count * sizeof(T) <= available && count * sizeof(T) <= static_cast((std::numeric_limits::max)()))) + { + std::memcpy(dest, first, count * sizeof(T)); + streambuf_access::bump(sb, static_cast(count * sizeof(T))); + return count * sizeof(T); + } #endif auto res = static_cast(sb->sgetn(reinterpret_cast(dest), static_cast(count * sizeof(T)))); if (JSON_HEDLEY_UNLIKELY(res < count * sizeof(T))) @@ -8886,7 +8960,7 @@ class input_stream_adapter // last one if @a drop_last. void copy_token(std::vector& out, bool drop_last) const { - out = token_carry; + token_carry.copy_to(out); if (token_segment != nullptr) { out.insert(out.end(), token_segment, current_end()); @@ -8899,6 +8973,47 @@ class input_stream_adapter #endif private: +#if !JSON_PRECISE_STREAM_POSITION + // the get area is exhausted, or there is none +#if !JSON_DISABLE_STREAM_BULK_SCAN + JSON_HEDLEY_NEVER_INLINE +#endif + std::char_traits::int_type get_character_slow() + { +#if !JSON_DISABLE_STREAM_BULK_SCAN + // sbumpc() refills the get area if it is exhausted + const bool refill = streambuf_access::get_ptr(sb) == streambuf_access::get_end(sb); + if (JSON_HEDLEY_UNLIKELY(refill)) + { + save_token_segment(); + } +#endif + + auto res = sb->sbumpc(); + // set eof manually, as we don't use the istream interface. + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) + { +#if !JSON_DISABLE_STREAM_BULK_SCAN + current_in_get_area = false; +#endif + is->clear(is->rdstate() | std::ios::eofbit); + } +#if !JSON_DISABLE_STREAM_BULK_SCAN + else if (JSON_HEDLEY_UNLIKELY(refill)) + { + // the consumed character is at gptr() - 1, unless the streambuf + // has no get area and returned it from uflow() + const char* const next = streambuf_access::get_ptr(sb); + after_refill(res, next != nullptr && next != streambuf_access::get_begin(sb)); + } + else + { + current_in_get_area = true; + } +#endif + return res; + } +#endif #if !JSON_DISABLE_STREAM_BULK_SCAN // Reads the protected get-area pointers of any std::streambuf. Naming the // members through a derived class makes forming the member pointers @@ -8922,6 +9037,11 @@ class input_stream_adapter { (s->*(&streambuf_access::gbump))(n); } + // sbumpc() for an exhausted get area + static std::streambuf::int_type next_character(std::streambuf* s) + { + return (s->*(&streambuf_access::uflow))(); + } }; // one past the character last returned by get_character(), if that is in @@ -8942,16 +9062,53 @@ class input_stream_adapter { if (token_segment != nullptr) { - token_carry.insert(token_carry.end(), token_segment, streambuf_access::get_end(sb)); + token_carry.append(token_segment, streambuf_access::get_end(sb)); token_segment = nullptr; } } +#if !JSON_PRECISE_STREAM_POSITION + // The streambuf has no get area (e.g., std::cin synchronized with stdio): + // every character comes from uflow(), and the token is kept by value. + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_no_get_area() + { + // the get area is exhausted, so this is what sbumpc() does + const auto res = streambuf_access::next_character(sb); + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof() || streambuf_access::get_end(sb) != nullptr)) + { + return get_character_no_get_area_other(res); + } + // current_in_get_area is false: only the first character of a get + // area, or the end of the input, is read here + if (tracking_token) + { + token_carry.push_back(std::char_traits::to_char_type(res)); + } + return res; + } + + // @a res came from uflow() of a streambuf without a get area: it is the + // end of the input, or the streambuf has a get area now + JSON_HEDLEY_NEVER_INLINE std::char_traits::int_type get_character_no_get_area_other(std::char_traits::int_type res) + { + if (res == std::char_traits::eof()) + { + current_in_get_area = false; + is->clear(is->rdstate() | std::ios::eofbit); + return res; + } + // the consumed character is at gptr() - 1 + after_refill(res, streambuf_access::get_ptr(sb) != streambuf_access::get_begin(sb)); + return res; + } +#endif + // character @a c was just read into a new get area (@a in_get_area) or // returned without one void after_refill(std::char_traits::int_type c, bool in_get_area) { current_in_get_area = in_get_area; + no_get_area = !in_get_area; if (!tracking_token) { return; @@ -8990,13 +9147,16 @@ class input_stream_adapter #endif #if !JSON_DISABLE_STREAM_BULK_SCAN /// the part of the current token that was in earlier get areas - std::vector token_carry{}; // NOLINT(readability-redundant-member-init) + token_carry_buffer token_carry{}; // NOLINT(readability-redundant-member-init) /// start of the current token's part in the get area, or nullptr const char* token_segment = nullptr; /// whether the character last returned by get_character() is in the get area bool current_in_get_area = false; /// whether a token is tracked at all; the binary readers never start one bool tracking_token = false; + /// whether the streambuf had no get area when it was last asked for a + /// character + bool no_get_area = false; #endif }; #endif // JSON_NO_IO @@ -9693,9 +9853,13 @@ NLOHMANN_JSON_NAMESPACE_END -#include // find_if, min +#include // copy, find_if, min +#include // array #include +#include // hash #include // numeric_limits +#include // unique_ptr +#include // operator new #include // string #include // enable_if_t #include // move, pair @@ -9720,6 +9884,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // snprintf #include // memcmp #include // initializer_list +#include // numeric_limits #include // char_traits, string #include // move #include // vector @@ -14068,6 +14233,15 @@ scan_number_done: /// significand (see scan_number_bulk_contiguous()) bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept { + // accept() only needs to know that the value is finite (see the parser): + // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a + // value that underflows is a finite number as well. Only a token that + // may overflow has to be converted to find out. + if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + { + value_float = 0; + return true; + } value_float = decimal_to_float(s); return true; } @@ -14884,10 +15058,66 @@ scan_number_done: } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // conditional, is_same + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +/*! +@brief Default base class of the @ref basic_json class. + +So that the correct implementations of the copy / move ctors / assign operators +of @ref basic_json do not require complex case distinctions +(no base class / custom base class used as customization point), +@ref basic_json always has a base class. +By default, this class is used because it is empty and thus has no effect +on the behavior of @ref basic_json. + +@note This class intentionally lives in namespace @ref nlohmann rather than + @ref nlohmann::detail. Every @ref basic_json specialization derives from + it (via @ref detail::json_base_class) unless a custom base class is + supplied, which makes its namespace an associated namespace of + @ref basic_json for the purpose of argument-dependent lookup (ADL). If + it lived in `nlohmann::detail`, that namespace - and with it the + library's internal `to_json`/`from_json` overloads - would leak into + ADL for any unqualified `to_json`/`from_json` call a user makes + involving a @ref basic_json argument, silently shadowing the user's own + overloads in some cases. +*/ +struct json_default_base {}; + +namespace detail +{ + +template +using json_base_class = typename std::conditional < + std::is_same::value, + ::nlohmann::json_default_base, + T + >::type; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include // #include +// #include + // #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -15184,20 +15414,114 @@ struct diagnostic_positions }; #endif +/*! +@brief a hash table that finds the position of a key in an object whose + elements are kept in a vector (ordered_map) + +The elements stay where they are: the table only holds their hashes and +positions, and the caller compares the keys. +*/ +class object_key_index +{ + public: + static constexpr std::size_t npos = static_cast(-1); + + /// @param[in] element_hashes the hashes of the keys of the elements that are there already + explicit object_key_index(std::vector&& element_hashes) + : hashes(std::move(element_hashes)) + { + std::size_t size = 256; + while (size < 4 * hashes.size()) + { + size *= 2; + } + rebuild(size); + } + + /// @return the position of the element with @a hash for which @a matches(position) holds, or npos + template + std::size_t find(const std::size_t hash, Matches matches) const + { + for (std::size_t slot = hash & mask;; slot = (slot + 1) & mask) + { + const std::size_t entry = table[slot]; + if (entry == 0) + { + return npos; + } + if (hashes[entry - 1] == hash && matches(entry - 1)) + { + return entry - 1; + } + } + } + + /// add the next element, whose position is the number of elements so far + void add(const std::size_t hash) + { + hashes.push_back(hash); + if (hashes.size() * 2 > table.size()) + { + rebuild(table.size() * 2); + } + else + { + place(hash, hashes.size()); + } + } + + private: + void rebuild(const std::size_t size) + { + table.assign(size, 0); + mask = size - 1; + for (std::size_t i = 0; i < hashes.size(); ++i) + { + place(hashes[i], i + 1); + } + } + + void place(const std::size_t hash, const std::size_t entry) + { + std::size_t slot = hash & mask; + while (table[slot] != 0) + { + slot = (slot + 1) & mask; + } + table[slot] = entry; + } + + /// the hash of each element, by position + std::vector hashes; + /// open addressing, a power of two in size: position + 1 of an element, or 0 for a free slot + std::vector table {}; // NOLINT(readability-redundant-member-init) + std::size_t mask = 0; +}; + /*! @brief SAX implementation to create a JSON value from SAX events This class implements the @ref json_sax interface and processes the SAX events to create a JSON value which makes it basically a DOM parser. The structure or -hierarchy of the JSON value is managed by the stack `ref_stack` which contains -a pointer to the respective array or object for each recursion depth. +hierarchy of the JSON value is managed by the stack `frames` which contains +an entry for each open array or object. The parser of the text formats, for +which the size of a container is not known up front, collects the elements of +an array and the members of an `ordered_json` object on a stack and moves them +into a container of the exact size (with @a Collect, as basic_json's own parse +functions use it); the binary readers use the plain +@ref json_sax_dom_binary_parser instead. After successful parsing, the value that is passed by reference to the -constructor contains the parsed value. +constructor contains the parsed value. Without @a Collect, every value is +stored in place as it arrives, so after a parse error the value holds what was +read up to it. @tparam BasicJsonType the JSON type +@tparam Collect whether to collect the values of containers of unknown size on + a stack (only for a caller that does not use the value after a + parse error) */ -template +template class json_sax_dom_parser { public: @@ -15270,6 +15594,930 @@ class json_sax_dom_parser return true; } + bool start_object(std::size_t len) + { +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS + // the values need their final place from the start (their parent + // pointers and positions refer to it), so no buffering + return start_stored_object(len); +#else + return start_object(len, std::integral_constant {}); +#endif + } + + bool key(string_t& val) + { + JSON_ASSERT(!frames.empty()); + return key(val, std::integral_constant {}); + } + + bool end_object() + { + JSON_ASSERT(!frames.empty()); + return end_object(std::integral_constant {}); + } + + bool start_array(std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > array_t().max_size())) + { + // the array does not exist yet: report with an empty one + BasicJsonType too_large(BasicJsonType::value_t::array); + return container_too_large_error(*this, len, "array", &too_large); + } + +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS + // the values need their final place from the start (their parent + // pointers and positions refer to it), so no buffering + start_stored_array(len); +#else + if (!collects(len)) + { + // the number of elements is known (or the elements must be in + // place as they arrive): store them directly + start_stored_array(len); + } + else + { + // the number of elements is not known, so growing the array + // would reallocate it (and, being a vector, move its elements) + // again and again. Collect the elements on a stack shared by all + // arrays instead, and copy them to an array of the exact size + // when the closing bracket arrives. + BasicJsonType* slot = nullptr; + std::size_t dest_member = 0; + container_destination(slot, dest_member); + reserve_elements(); + frames.push_back(frame{nullptr, nullptr, slot, elements.size(), ordered_state(dest_member, false)}); + in_object = false; + stored_array = nullptr; + } +#endif + return true; + } + + bool end_array() + { + JSON_ASSERT(!frames.empty()); + JSON_ASSERT(frames.back().object == nullptr); + const frame f = frames.back(); + frames.pop_back(); + reenter_enclosing_container(); + + if (f.array == nullptr) + { + // the elements are the top of the shared elements stack, from + // f.start on (the elements below belong to enclosing arrays that + // are still open): move them into the array and pop them + const auto first = elements.begin() + static_cast(f.start); + auto* const arr = BasicJsonType::template create(std::make_move_iterator(first), std::make_move_iterator(elements.end())); + BasicJsonType value; + value.m_data.m_value.array = arr; + value.m_data.m_type = BasicJsonType::value_t::array; + elements.erase(first, elements.end()); + + place_container(f, frames.empty() ? nullptr : &frames.back(), std::move(value), true); + return true; + } + +#if JSON_DIAGNOSTIC_POSITIONS + diagnostic_positions::set_container_end(*f.slot, m_lexer_ref); +#endif +#if JSON_DIAGNOSTICS + f.slot->set_parents(); +#endif + return true; + } + + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const Exception& ex) + { + errored = true; + static_cast(ex); + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + using object_t = typename BasicJsonType::object_t; + using array_t = typename BasicJsonType::array_t; + + /// whether objects keep their elements in a vector and a key can be hashed, + /// so that a big object can find the keys it has in a hash table instead of + /// searching all of them + template + using detect_indexable_keys = decltype(std::declval().emplace_new_key(std::declval(), std::declval()), + std::hash()(std::declval())); + static constexpr bool indexed_keys = is_detected::value; + + /// what a frame holds for objects that are kept in a vector: the key index + /// of an object with many keys, and for an object that is collected on the + /// stack the member of the enclosing object it will become + struct ordered_frame + { + /// the entry of key_indexes of an object that has one, else nullptr + object_key_index* index; + /// object or array without a slot in an object collected on the stack: + /// the index of the member in `members` it goes to + std::size_t dest_member; + /// whether this is an object collected on the stack + bool buffered; + }; + struct no_ordered_frame {}; + using ordered_frame_state = typename std::conditional::type; + + static ordered_frame make_ordered_state(const std::size_t dest_member, const bool buffered, std::true_type /*indexed*/) noexcept + { + ordered_frame state = {nullptr, dest_member, buffered}; + return state; + } + + static no_ordered_frame make_ordered_state(const std::size_t /*dest_member*/, const bool /*buffered*/, std::false_type /*indexed*/) noexcept + { + return no_ordered_frame(); + } + + static ordered_frame_state ordered_state(const std::size_t dest_member, const bool buffered) noexcept + { + return make_ordered_state(dest_member, buffered, std::integral_constant {}); + } + + /// an array or object that is being read + struct frame + { + /// the object, or nullptr for an array + object_t* object; + /// the array if its elements are stored in it as they arrive, or + /// nullptr for an array and an object + array_t* array; + /// array without `array`: where the finished array goes (nullptr: it + /// is an element of the enclosing array); otherwise: the value that + /// holds the object or array + BasicJsonType* slot; + /// array without `array`: the index of its first element in `elements` + std::size_t start; + /// what objects in a vector add (see ordered_frame) + ordered_frame_state ordered; + }; + + /// the open arrays and objects; the first few live in the object itself, so + /// that the common documents do not allocate for them + class frame_stack + { + public: + frame_stack() noexcept + : local() // NOLINT(readability-redundant-member-init) + {} + frame_stack(const frame_stack&) = delete; + frame_stack& operator=(const frame_stack&) = delete; + + frame_stack(frame_stack&& other) noexcept + : local() // NOLINT(readability-redundant-member-init) + { + take(other); + } + + frame_stack& operator=(frame_stack&& other) noexcept + { + if (this != &other) + { + release(); + take(other); + } + return *this; + } + + ~frame_stack() = default; + + bool empty() const noexcept + { + return count == 0; + } + + std::size_t size() const noexcept + { + return count; + } + + frame& back() noexcept + { + return data[count - 1]; + } + + frame& operator[](const std::size_t i) noexcept + { + return data[i]; + } + + void push_back(const frame& f) + { + if (JSON_HEDLEY_UNLIKELY(count == capacity)) + { + grow(); + } + data[count++] = f; + } + + void pop_back() noexcept + { + JSON_ASSERT(count > 0); + --count; + } + + private: + JSON_HEDLEY_NEVER_INLINE + void grow() + { + const std::size_t new_capacity = capacity * 2; + std::unique_ptr new_heap(new frame[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + std::copy(data, data + count, new_heap.get()); + heap = std::move(new_heap); + data = heap.get(); + capacity = new_capacity; + } + + void release() noexcept + { + heap.reset(); + data = local.data(); + capacity = local_capacity; + } + + /// make the contents of @a other our own + void take(frame_stack& other) noexcept + { + if (other.heap == nullptr) + { + std::copy(other.local.begin(), other.local.begin() + static_cast(other.count), local.begin()); + data = local.data(); + capacity = local_capacity; + } + else + { + heap = std::move(other.heap); + data = heap.get(); + capacity = other.capacity; + other.data = other.local.data(); + other.capacity = local_capacity; + } + count = other.count; + other.count = 0; + } + + static constexpr std::size_t local_capacity = 8; + + /// the first frames, left uninitialized: a frame is written before it is + /// read (the user-provided constructor keeps the member initialization, + /// which -Weffc++ asks for, from zeroing them) + struct local_frames : std::array + { + local_frames() noexcept {} // NOLINT(modernize-use-equals-default,hicpp-use-equals-default,cppcoreguidelines-pro-type-member-init,hicpp-member-init): see above + }; + + local_frames local; + /// the frames once there are more than local_capacity of them + std::unique_ptr heap {}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init) + frame* data = local.data(); + std::size_t count = 0; + std::size_t capacity = local_capacity; + }; + + void release_key_index(const frame& f, std::true_type /*indexed*/) + { + if (JSON_HEDLEY_UNLIKELY(f.ordered.index != nullptr)) + { + ordered_data.key_indexes.pop_back(); + } + } + + static void release_key_index(const frame& /*f*/, std::false_type /*indexed*/) noexcept {} + + /// the number of keys from which an object looks them up in a hash table + static constexpr std::size_t key_index_threshold = 64; + + /// add @a key to the object of @a f (or find it, if it is there), searching it in std::map + static BasicJsonType* insert_key(frame& f, string_t&& key, std::false_type /*indexed*/) + { + return &f.object->operator[](std::move(key)); + } + + /// add @a key to the object of @a f (or find it, if it is there), searching it in a vector + BasicJsonType* insert_key(frame& f, string_t&& key, std::true_type /*indexed*/) + { + object_t& obj = *f.object; + if (JSON_HEDLEY_LIKELY(obj.size() < key_index_threshold)) + { + return &obj[std::move(key)]; + } + return insert_key_with_index(f, std::move(key)); + } + + /// add @a key to the big object of @a f (or find it, if it is there), using its key index + JSON_HEDLEY_NEVER_INLINE + BasicJsonType* insert_key_with_index(frame& f, string_t&& key) + { + object_t& obj = *f.object; + const std::hash hasher {}; + const std::size_t hash = hasher(key); + if (JSON_HEDLEY_UNLIKELY(f.ordered.index == nullptr)) + { + std::vector hashes; + hashes.reserve(obj.size()); + for (const auto& element : obj) + { + hashes.push_back(hasher(element.first)); + } + ordered_data.key_indexes.emplace_back(new object_key_index(std::move(hashes))); + f.ordered.index = ordered_data.key_indexes.back().get(); + } + + const std::size_t position = f.ordered.index->find(hash, [&obj, &key](std::size_t i) + { + return (obj.begin() + static_cast(i))->first == key; + }); + if (position != object_key_index::npos) + { + return &(obj.begin() + static_cast(position))->second; + } + + obj.emplace_new_key(key, BasicJsonType()); + f.ordered.index->add(hash); + return &obj.back().second; + } + + /// where an array or object that starts now will go when it is finished: + /// @a slot is the value to assign it to (the root or a member of an object + /// that is stored directly), or nullptr if it is an element of the + /// enclosing array or a member of the enclosing object that is collected on + /// the stack, in which case @a dest_member is the member + void container_destination(BasicJsonType*& slot, std::size_t& dest_member) + { + if (frames.empty()) + { + slot = &root; + } + else if (frames.back().object != nullptr) + { + slot = object_element; + } + else + { + collected_object_member(frames.back(), dest_member, std::integral_constant {}); + } + } + + void collected_object_member(const frame& enclosing, std::size_t& dest_member, std::true_type /*indexed*/) const noexcept + { + if (enclosing.ordered.buffered) + { + dest_member = ordered_data.member_index; + } + } + + static void collected_object_member(const frame& /*enclosing*/, std::size_t& /*dest_member*/, std::false_type /*indexed*/) noexcept {} + + static bool is_collected_object(const frame& f, std::true_type /*indexed*/) noexcept + { + return f.ordered.buffered; + } + + static bool is_collected_object(const frame& /*f*/, std::false_type /*indexed*/) noexcept + { + return false; + } + + /// put a finished array or object that was collected on the stack, or + /// that outgrew it, where it belongs; @a enclosing is the container it is in. + /// @a enclosing_is_innermost tells whether that is the container that gets + /// the next value, so that it can be switched to storing its elements + /// directly if the value was the one too many. + void place_container(const frame& f, frame* enclosing, BasicJsonType&& value, const bool enclosing_is_innermost) + { + if (f.slot != nullptr) + { + *f.slot = std::move(value); + } + else if (is_collected_object(*enclosing, std::integral_constant {})) + { + place_member(f, std::move(value), std::integral_constant {}); + } + else if (enclosing->array != nullptr) + { + enclosing->array->emplace_back(std::move(value)); + } + else + { + elements.emplace_back(std::move(value)); + if (JSON_HEDLEY_UNLIKELY(enclosing_is_innermost && elements.size() - enclosing->start > buffer_limit)) + { + spill(*enclosing); + } + } + } + + void place_member(const frame& f, BasicJsonType&& value, std::true_type /*indexed*/) + { + ordered_data.members[f.ordered.dest_member].second = std::move(value); + } + + static void place_member(const frame& /*f*/, BasicJsonType&& /*value*/, std::false_type /*indexed*/) noexcept {} + + /// store the members of a new object directly in it + bool start_stored_object(std::size_t len) + { + BasicJsonType* const ref = store(BasicJsonType::value_t::object); + frames.push_back(frame{ref->m_data.m_value.object, nullptr, ref, 0, ordered_state(0, false)}); + in_object = true; + stored_array = nullptr; + +#if JSON_DIAGNOSTIC_POSITIONS + // Ensure this is after the call to store to ensure correct start position. + diagnostic_positions::set_container_start(*ref, m_lexer_ref); +#endif + + return check_container_size(*this, len, "object", ref); + } + + /// objects in std::map and the like have no use for collecting their members + bool start_object(std::size_t len, std::false_type /*indexed*/) + { + return start_stored_object(len); + } + + /// An object in a vector of members adds a member by appending it to the + /// vector, which grows it again and again (and copies the keys each time). + /// Collect the members of an object of unknown size on a stack shared by + /// all open objects instead, and move them into a vector of the exact size + /// when the closing brace arrives. + bool start_object(std::size_t len, std::true_type /*indexed*/) + { + if (!collects(len)) + { + return start_stored_object(len); + } + + BasicJsonType* slot = nullptr; + std::size_t dest_member = 0; + container_destination(slot, dest_member); + frames.push_back(frame{nullptr, nullptr, slot, ordered_data.members.size(), ordered_state(dest_member, true)}); + in_object = false; + stored_array = nullptr; + return true; + } + + bool key(string_t& val, std::false_type /*indexed*/) + { + JSON_ASSERT(frames.back().object != nullptr); + + // add null at the given key and store the reference for later; the + // key is moved, as the lexer starts the next token with reset() + object_element = insert_key(frames.back(), std::move(val), std::false_type {}); + return true; + } + + bool key(string_t& val, std::true_type /*indexed*/) + { + frame& f = frames.back(); + if (JSON_HEDLEY_UNLIKELY(f.object == nullptr)) + { + JSON_ASSERT(f.ordered.buffered); + if (collect_member(f, val)) + { + return true; + } + // the object has grown big: it is stored directly from here on + } + object_element = insert_key(f, std::move(val), std::true_type {}); + return true; + } + + /*! + @brief add a member to the object that is collected on the stack, or find the one with the key + @return false if the object has collected enough members to be stored + directly, which is done and which leaves the key to be inserted + */ + bool collect_member(frame& f, string_t& key) + { + for (std::size_t i = f.start; i < ordered_data.members.size(); ++i) + { + if (ordered_data.members[i].first == key) + { + ordered_data.member_index = i; + return true; + } + } + + if (JSON_HEDLEY_UNLIKELY(ordered_data.members.size() - f.start >= object_buffer_limit)) + { + spill_object(f); + return false; + } + + reserve_members(); + ordered_data.members.emplace_back(std::move(key), BasicJsonType()); + ordered_data.member_index = ordered_data.members.size() - 1; + return true; + } + + /// the members of an object that is collected on the stack as an object; + /// they are the top of the shared member stack, from f.start on (the + /// members below belong to enclosing objects that are still open) + object_t* members_to_object(const frame& f, BasicJsonType& value) + { + const auto first = ordered_data.members.begin() + static_cast(f.start); + auto* const obj = BasicJsonType::template create(std::make_move_iterator(first), std::make_move_iterator(ordered_data.members.end())); + value.m_data.m_value.object = obj; + value.m_data.m_type = BasicJsonType::value_t::object; + ordered_data.members.erase(first, ordered_data.members.end()); + return obj; + } + + /// switch an object that collects its members on the stack to storing them directly + JSON_HEDLEY_NEVER_INLINE + void spill_object(frame& f) + { + BasicJsonType value; + f.object = members_to_object(f, value); + f.ordered.buffered = false; + in_object = true; + // false positive: an object without an enclosing container is the root, + // which has a slot, so place_container() does not use the nullptr + // @infer-ignore NULLPTR_DEREFERENCE + place_container(f, frames.size() >= 2 ? &frames[frames.size() - 2] : nullptr, std::move(value), false); + } + + bool end_object(std::false_type /*indexed*/) + { + return end_stored_object(); + } + + bool end_object(std::true_type /*indexed*/) + { + if (JSON_HEDLEY_LIKELY(frames.back().object != nullptr)) + { + return end_stored_object(); + } + + // the members are on the stack: make them the object + const frame f = frames.back(); + frames.pop_back(); + reenter_enclosing_container(); + + BasicJsonType value; + members_to_object(f, value); + place_container(f, frames.empty() ? nullptr : &frames.back(), std::move(value), true); + return true; + } + + bool end_stored_object() + { + JSON_ASSERT(frames.back().object != nullptr); + +#if JSON_DIAGNOSTICS || JSON_DIAGNOSTIC_POSITIONS + // objects are never collected, so their value is where it was stored + BasicJsonType* const ref = frames.back().slot; +#endif +#if JSON_DIAGNOSTIC_POSITIONS + diagnostic_positions::set_container_end(*ref, m_lexer_ref); +#endif +#if JSON_DIAGNOSTICS + ref->set_parents(); +#endif + + release_key_index(frames.back(), std::integral_constant {}); + frames.pop_back(); + reenter_enclosing_container(); + return true; + } + + /// the number of members an object collects on the stack before it + /// switches to storing them directly + static constexpr std::size_t object_buffer_limit = 32; + + void reserve_members() + { + if (JSON_HEDLEY_UNLIKELY(ordered_data.members.capacity() == 0)) + { + ordered_data.members.reserve(16); + } + } + + /// the number of elements an array collects on the stack before it + /// switches to storing them directly, so that it does not hold up the + /// stack with its elements (and move all of them in the end) + static constexpr std::size_t buffer_limit = 16; + + void reserve_elements() + { + if (JSON_HEDLEY_UNLIKELY(elements.capacity() == 0)) + { + elements.reserve(32); + } + } + + /// store the elements of a new array directly in it + void start_stored_array(const std::size_t len) + { + BasicJsonType* const ref = store(BasicJsonType::value_t::array); + frames.push_back(frame{nullptr, ref->m_data.m_value.array, ref, 0, ordered_state(0, false)}); + in_object = false; + stored_array = ref->m_data.m_value.array; + +#if JSON_DIAGNOSTIC_POSITIONS + // Ensure this is after the call to store to ensure correct start position. + diagnostic_positions::set_container_start(*ref, m_lexer_ref); +#endif + + if (len != detail::unknown_size()) + { + reserve_array(*ref->m_data.m_value.array, len, priority_tag<1> {}); + } + } + + /*! + @brief store a value as the next element of the open array or object (or + as the root) + @return the stored value + */ + template + JSON_HEDLEY_RETURNS_NON_NULL + BasicJsonType* store(Value&& v) + { + if (stored_array != nullptr) + { + stored_array->emplace_back(std::forward(v)); + return &stored_array->back(); + } + + if (in_object) + { + JSON_ASSERT(object_element); + set_member(*object_element, std::forward(v), std::is_same {}); + return object_element; + } + + return store_collected(std::forward(v), std::integral_constant {}); + } + + /// store a value that is the root, an element of an array that is collected + /// on the stack, or (in a vector) a member of an object that is + template + BasicJsonType* store_collected(Value&& v, std::true_type /*indexed*/) + { + if (!frames.empty() && frames.back().ordered.buffered) + { + BasicJsonType& member = ordered_data.members[ordered_data.member_index].second; + set_member(member, std::forward(v), std::is_same {}); + return &member; + } + return store_collected(std::forward(v), std::false_type {}); + } + + template + BasicJsonType* store_collected(Value&& v, std::false_type /*indexed*/) + { + if (frames.empty()) + { + root = BasicJsonType(std::forward(v)); + return &root; + } + + frame& f = frames.back(); + elements.emplace_back(std::forward(v)); + if (JSON_HEDLEY_UNLIKELY(elements.size() - f.start > buffer_limit)) + { + return spill(f); + } + return &elements.back(); + } + + /*! + @brief switch an array that collects its elements on the stack to storing + them directly + @return the last element of the array + */ + JSON_HEDLEY_NEVER_INLINE + BasicJsonType* spill(frame& f) + { + JSON_ASSERT(f.array == nullptr); + const auto first = elements.begin() + static_cast(f.start); + BasicJsonType value(BasicJsonType::value_t::array); + array_t& arr = *value.m_data.m_value.array; + // the capacity a vector that grew by itself has with this many elements + reserve_array(arr, 2 * buffer_limit, priority_tag<1> {}); + arr.insert(arr.end(), std::make_move_iterator(first), std::make_move_iterator(elements.end())); + elements.erase(first, elements.end()); + f.array = &arr; + stored_array = &arr; + + // the array takes the place its first element had on the stack + place_container(f, frames.size() >= 2 ? &frames[frames.size() - 2] : nullptr, std::move(value), false); + return &arr.back(); + } + + /// set the value of the member key() just added + template + JSON_HEDLEY_ALWAYS_INLINE + static void set_member(BasicJsonType& member, Value&& v, std::true_type /*default base*/) + { + if (JSON_HEDLEY_LIKELY(member.is_null())) + { + // a new member is null and holds nothing: construct the value + // over it instead of destroying and swapping + ::new (static_cast(&member)) BasicJsonType(std::forward(v)); + } + else + { + // the key occurred before + member = BasicJsonType(std::forward(v)); + } + } + + /// a custom base class may care about the lifetime of the value + template + static void set_member(BasicJsonType& member, Value&& v, std::false_type /*custom base*/) + { + member = BasicJsonType(std::forward(v)); + } + + /// whether the values of a container of @a len elements are collected on the stack + static bool collects(const std::size_t len) noexcept + { + return collects(len, std::integral_constant {}); + } + + static bool collects(const std::size_t len, std::true_type /*collect*/) noexcept + { + return len == detail::unknown_size(); + } + + static bool collects(const std::size_t /*len*/, std::false_type /*collect*/) noexcept + { + return false; + } + + /// after a container was closed: the one it was in is the one that gets the next value + void reenter_enclosing_container() + { + if (frames.empty()) + { + in_object = false; + stored_array = nullptr; + } + else + { + const frame& f = frames.back(); + in_object = f.object != nullptr; + stored_array = f.array; + } + } + + template + void handle_value(Value&& v) + { +#if JSON_DIAGNOSTIC_POSITIONS + BasicJsonType* const stored = store(std::forward(v)); + diagnostic_positions::set_from_lexer(*stored, m_lexer_ref); +#else + store(std::forward(v)); +#endif + } + + /// the parsed JSON value + BasicJsonType& root; + /// the open arrays and objects + frame_stack frames {}; + /// the elements of the open arrays that are not stored in them yet + std::vector elements {}; + /// what objects in a vector need besides the frames (std::map based objects + /// need nothing, so they do not carry it) + struct ordered_data_t + { + /// the members of the open objects in a vector that are not stored in them yet + std::vector> members {}; + /// the member of the innermost such object that key() added or found last + std::size_t member_index = 0; + /// the key indexes of the open objects that have one, innermost last + std::vector> key_indexes {}; // NOLINT(readability-redundant-member-init) + }; + struct no_ordered_data_t {}; + typename std::conditional::type ordered_data {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether the innermost container is an object (frames.back().object) + bool in_object = false; + /// the array that stores the elements of the innermost array as they + /// arrive (frames.back().array), or nullptr + array_t* stored_array = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// the lexer reference to obtain the current position + lexer_t* m_lexer_ref = nullptr; +}; + +/*! +@brief SAX implementation to create a JSON value from the events of a binary reader + +The DOM parser of the text formats (json_sax_dom_parser) collects the elements +of arrays and the members of objects on a stack and builds them in one +exact-size allocation, which pays off because a text parser does not know the +number of elements up front. Binary formats announce the size of their +containers (CBOR, MessagePack, BSON and BON8 do, and so do the optimized +containers of UBJSON and BJData), so they store each value directly in its +container, and the bookkeeping of the collecting parser would be pure +overhead for them. This is the plain parser: the structure or hierarchy of the +JSON value is managed by the stack `ref_stack` which contains a pointer to the +respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template +class json_sax_dom_binary_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using lexer_t = lexer; + + /*! + @param[in,out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_binary_parser(BasicJsonType& r, const bool allow_exceptions_ = true, lexer_t* lexer_ = nullptr) + : root(r), allow_exceptions(allow_exceptions_), m_lexer_ref(lexer_) + {} + + // make class move-only + json_sax_dom_binary_parser(const json_sax_dom_binary_parser&) = delete; + json_sax_dom_binary_parser(json_sax_dom_binary_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + json_sax_dom_binary_parser& operator=(const json_sax_dom_binary_parser&) = delete; + json_sax_dom_binary_parser& operator=(json_sax_dom_binary_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~json_sax_dom_binary_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); + return true; + } + + bool binary(binary_t& val) + { + // json_sax documents that the passed value may be moved from, + // so hand the buffer over instead of copying it + handle_value(std::move(val)); + return true; + } + bool start_object(std::size_t len) { ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); @@ -15532,16 +16780,13 @@ class json_sax_dom_callback_parser // handle_value() without touching the key stacks if (keep_stack.back()) { - BasicJsonType k = BasicJsonType(val); - static_cast(callback(static_cast(ref_stack.size()), parse_event_t::key, k)); + static_cast(callback(static_cast(ref_stack.size()), parse_event_t::key, key_value(val))); } return true; } - BasicJsonType k = BasicJsonType(val); - // check callback for the key - const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, key_value(val)); key_keep_stack.push_back(keep); // remember the key so a rejected value can be erased without searching // the object for it (kept in lockstep with key_keep_stack) @@ -15552,7 +16797,7 @@ class json_sax_dom_callback_parser { auto& obj = *ref_stack.back()->m_data.m_value.object; // a single lookup: insert the placeholder, or find the existing member - const auto inserted = obj.emplace(val, discarded); + const auto inserted = obj.emplace(std::move(val), discarded); const auto it = inserted.first; if (!inserted.second) { @@ -15726,6 +16971,31 @@ class json_sax_dom_callback_parser private: + /*! + @brief the value of a key event for the callback + + A string value of its own for every key would allocate it every time, so + one is kept and set to the key. A callback that changed or moved it gets a + new one, as it would have got a new value for every key anyway. + */ + BasicJsonType& key_value(const string_t& key) + { +#if JSON_DIAGNOSTIC_POSITIONS + // a callback may have left positions on the value + key_scratch = BasicJsonType(key); +#else + if (JSON_HEDLEY_LIKELY(key_scratch.is_string())) + { + *key_scratch.m_data.m_value.string = key; + } + else + { + key_scratch = BasicJsonType(key); + } +#endif + return key_scratch; + } + /// if there is a pending duplicate-key stash entry for this exact slot, /// remove it from the stash; if restore_value is true, the stashed /// previous value is moved back into the slot first (use this when the @@ -15973,6 +17243,8 @@ class json_sax_dom_callback_parser const bool allow_exceptions = true; /// a discarded value for the callback BasicJsonType discarded = BasicJsonType::value_t::discarded; + /// the value of the key event for the callback (see key_value()) + BasicJsonType key_scratch {}; /// the lexer reference to obtain the current position lexer_t* m_lexer_ref = nullptr; }; @@ -16255,7 +17527,7 @@ JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 2 /*! @brief deserialization of BJData, BON8, BSON, CBOR, MessagePack, and UBJSON values */ -template> +template> class binary_reader { using number_integer_t = typename BasicJsonType::number_integer_t; @@ -20712,8 +21984,8 @@ using parser_callback_t = template struct sax_ignores_number_text : std::false_type {}; -template -struct sax_ignores_number_text> : std::true_type {}; +template +struct sax_ignores_number_text> : std::true_type {}; template struct sax_ignores_number_text> : std::true_type {}; @@ -20787,7 +22059,7 @@ class parser } else { - json_sax_dom_parser sdp(result, allow_exceptions, &m_lexer); + json_sax_dom_parser sdp(result, allow_exceptions, &m_lexer); // in case of an error, return a discarded value if (!parse_dom(sdp, strict)) @@ -20899,12 +22171,19 @@ class parser JSON_HEDLEY_NON_NULL(2) bool sax_parse_internal(SAX* sax) { + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; + // stack to remember the hierarchy of structured values we are parsing // true = array; false = object (bytes, not std::vector, whose // push_back() is a call with GCC) std::vector states; - // value to avoid a goto (see comment where set to true) - bool skip_to_state_evaluation = false; + // a document that starts with a container pushes onto the stack at + // once: allocate it in one step instead of growing it from 1 byte + if (last_token == token_type::begin_object || last_token == token_type::begin_array) + { + states.reserve(32); + } while (true) { @@ -22286,58 +23565,6 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // conditional, is_same - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN - -/*! -@brief Default base class of the @ref basic_json class. - -So that the correct implementations of the copy / move ctors / assign operators -of @ref basic_json do not require complex case distinctions -(no base class / custom base class used as customization point), -@ref basic_json always has a base class. -By default, this class is used because it is empty and thus has no effect -on the behavior of @ref basic_json. - -@note This class intentionally lives in namespace @ref nlohmann rather than - @ref nlohmann::detail. Every @ref basic_json specialization derives from - it (via @ref detail::json_base_class) unless a custom base class is - supplied, which makes its namespace an associated namespace of - @ref basic_json for the purpose of argument-dependent lookup (ADL). If - it lived in `nlohmann::detail`, that namespace - and with it the - library's internal `to_json`/`from_json` overloads - would leak into - ADL for any unqualified `to_json`/`from_json` call a user makes - involving a @ref basic_json argument, silently shadowing the user's own - overloads in some cases. -*/ -struct json_default_base {}; - -namespace detail -{ - -template -using json_base_class = typename std::conditional < - std::is_same::value, - ::nlohmann::json_default_base, - T - >::type; - -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ @@ -22350,14 +23577,14 @@ NLOHMANN_JSON_NAMESPACE_END -#include // all_of +#include // all_of, copy #include // array #include // isdigit #include // errno, ERANGE #include // size_t #include // uint64_t #include // strtoull -#include // memcmp, memcpy +#include // memchr, memcmp, memcpy #ifndef JSON_NO_IO #include // ostream #endif // JSON_NO_IO @@ -22430,14 +23657,38 @@ class json_pointer /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ string_t to_string() const { + if (reference_tokens.empty()) + { + return string_t(); + } + // one buffer: every token adds its escaped form (at least its length) and a slash std::size_t length = 0; + bool plain = true; for (const auto& reference_token : reference_tokens) { length += reference_token.size() + 1; + const char* const data = reference_token.data(); + for (std::size_t i = 0, size = reference_token.size(); plain && i < size; ++i) + { + plain = data[i] != '~' && data[i] != '/'; + } } string_t result; + if (JSON_HEDLEY_LIKELY(plain)) + { + // nothing to escape: the tokens are copied into the string + result.resize(length); + char* out = &result[0]; + for (const auto& reference_token : reference_tokens) + { + *out++ = '/'; + out = std::copy(reference_token.data(), reference_token.data() + reference_token.size(), out); + } + return result; + } + result.reserve(length); for (const auto& reference_token : reference_tokens) { @@ -22653,6 +23904,12 @@ class json_pointer exceeds_size_type ///< @a s converts to an integer that exceeds size_type }; + /// whether a reference token is "-" (a plain comparison with a string literal is not always inlined) + static bool is_dash(const string_t& s) noexcept + { + return s.size() == 1 && s[0] == '-'; + } + /*! @param[in] s reference token to be converted into an array index @param[out] idx the integer representation of @a s if @ref array_index_status::ok @@ -22678,7 +23935,7 @@ class json_pointer // error condition (cf. RFC 6901, Sect. 4); this also covers single- // character tokens, so "/x" and "/xy" fail alike; "-" and the empty // token are left to the conversion below and are reported as unresolved - if (JSON_HEDLEY_UNLIKELY(!s.empty() && s != "-" && !(s[0] >= '0' && s[0] <= '9'))) + if (JSON_HEDLEY_UNLIKELY(!s.empty() && !is_dash(s) && !(s[0] >= '0' && s[0] <= '9'))) { return array_index_status::not_a_number; } @@ -22825,9 +24082,40 @@ class json_pointer return pos == size; } + /// reserve room for @a n elements of an array type that has reserve() + template + static auto reserve_elements(ArrayType& array, const std::size_t n, detail::priority_tag<1> /*unused*/) + -> decltype(array.reserve(n), void()) + { + array.reserve(n); + } + + template + static void reserve_elements(ArrayType& /*array*/, const std::size_t /*n*/, detail::priority_tag<0> /*unused*/) + {} + /// @brief object member access with a key that is only copied if it is inserted + /// + /// Objects that can find the position of a key and insert there + /// (std::map) search once, others look the key up and add it if it is missing. template - static BasicJsonType& object_member(BasicJsonType& object, const string_t& key) + static auto object_member_impl(BasicJsonType& object, const string_t& key, detail::priority_tag<1> /*unused*/) + -> decltype(object.m_data.m_value.object->emplace_hint(object.m_data.m_value.object->lower_bound(key), key, nullptr), std::declval()) + { + auto& members = *object.m_data.m_value.object; + auto it = members.lower_bound(key); + if (it != members.end() && !members.key_comp()(key, it->first)) + { + return it->second; + } + + const auto old_capacity = object.container_capacity(); + it = members.emplace_hint(it, key, nullptr); + return object.set_parent_after_insert(it->second, old_capacity); + } + + template + static BasicJsonType& object_member_impl(BasicJsonType& object, const string_t& key, detail::priority_tag<0> /*unused*/) { auto& members = *object.m_data.m_value.object; const auto it = members.find(key); @@ -22838,6 +24126,12 @@ class json_pointer return object.operator[](key); } + template + static BasicJsonType& object_member(BasicJsonType& object, const string_t& key) + { + return object_member_impl(object, key, detail::priority_tag<1> {}); + } + JSON_PRIVATE_UNLESS_TESTED: json_pointer top() const { @@ -23070,48 +24364,103 @@ class json_pointer } } + /*! + @brief the values along the pointer last resolved by @ref resolve_plain + + A patch from @ref basic_json::diff lists its operations in document order, + so consecutive pointers share most of their reference tokens; the values + along the previous pointer are kept to resume from the common prefix. + `steps[k]` is the value after the first `k` reference tokens together with + the length of the prefix of `*path` that holds them. A value stays valid as + long as no operation changes its container: callers keep only values up to + the parent of the pointer they modify, or call `reset()`. + */ + template + struct plain_steps + { + const string_t* path = nullptr; + std::vector> steps{}; + + void reset() noexcept + { + path = nullptr; + steps.clear(); + } + }; + /*! @brief resolve a plain JSON pointer without building its reference tokens - @param[in] ptr the value the pointer starts at + @param[in] root the value the pointer starts at @param[in] path a JSON pointer string @param[in] length the number of leading bytes of @a path to resolve; 0 or the position of a slash @param[in,out] token a string to hold the reference tokens, to reuse its storage + @param[in,out] cache the values along the previously resolved pointer; + @a path must outlive the next call @return the pointed to value, or `nullptr` if the pointer does not denote - an existing value, or does so in a way that is not plain: a - reference token with an escape, an array index that is malformed or - out of range, or a "-". No exception is thrown for these; callers - use @ref json_pointer instead, which handles them. + an existing value, or does so in a way that is not plain: an array + index that is malformed or out of range, or a "-", or a reference + token with an escape. No exception is + thrown for these; callers use @ref json_pointer instead, which + handles them. */ template - static BasicJsonType* resolve_plain(BasicJsonType* ptr, const string_t& path, const std::size_t length, string_t& token) + static BasicJsonType* resolve_plain(BasicJsonType* root, const string_t& path, const std::size_t length, string_t& token, plain_steps& cache) { - for (std::size_t pos = 0; pos < length;) + auto& steps = cache.steps; + + // resume from the deepest step that is also a prefix of this path + std::size_t depth = 1; + if (cache.path != nullptr && !steps.empty() && steps[0].second == root) + { + const string_t& previous = *cache.path; + for (depth = steps.size(); depth > 1; --depth) + { + // a step ends at a token boundary of the previous path + const std::size_t prefix = steps[depth - 1].first; + if (prefix <= length && (prefix == length || path[prefix] == '/') && std::memcmp(path.data(), previous.data(), prefix) == 0) + { + break; + } + } + steps.resize(depth); + } + else + { + steps.assign(1, std::make_pair(std::size_t{0}, root)); + } + cache.path = &path; + + BasicJsonType* ptr = steps[depth - 1].second; + for (std::size_t pos = steps[depth - 1].first; pos < length;) { if (JSON_HEDLEY_UNLIKELY(path[pos] != '/')) { return nullptr; } - // the reference token between the slash at pos and the next one - std::size_t end = path.find_first_of('/', pos + 1); - if (end == string_t::npos || end > length) + // the reference token between the slash at pos and the next one, + // which must not have an escape + const char* const begin = path.data() + pos + 1; + const char* const limit = path.data() + length; + const char* end = begin; + while (end < limit && *end != '/') { - end = length; - } - token.clear(); - token.append(path.data() + pos + 1, end - pos - 1); - if (JSON_HEDLEY_UNLIKELY(token.find_first_of('~') != string_t::npos)) - { - return nullptr; + if (JSON_HEDLEY_UNLIKELY(*end == '~')) + { + return nullptr; + } + ++end; } switch (ptr->type()) { case detail::value_t::object: { + token.clear(); + token.append(begin, static_cast(end - begin)); const auto it = ptr->m_data.m_value.object->find(token); if (JSON_HEDLEY_UNLIKELY(it == ptr->m_data.m_value.object->end())) { @@ -23123,8 +24472,24 @@ class json_pointer case detail::value_t::array: { - typename BasicJsonType::size_type idx{}; - if (JSON_HEDLEY_UNLIKELY(parse_array_index(token, idx) != array_index_status::ok || idx >= ptr->m_data.m_value.array->size())) + // an index of up to 18 digits without a leading zero fits into size_type + // (9 digits for a 32-bit size_type); everything else is left to json_pointer + constexpr std::size_t max_digits = sizeof(typename BasicJsonType::size_type) >= 8 ? 18 : 9; + const auto digits = static_cast(end - begin); + if (JSON_HEDLEY_UNLIKELY(digits == 0 || digits > max_digits || (digits > 1 && *begin == '0'))) + { + return nullptr; + } + typename BasicJsonType::size_type idx = 0; + for (const char* c = begin; c != end; ++c) + { + if (JSON_HEDLEY_UNLIKELY(*c < '0' || *c > '9')) + { + return nullptr; + } + idx = static_cast((idx * 10) + static_cast(*c - '0')); + } + if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size())) { return nullptr; } @@ -23143,7 +24508,8 @@ class json_pointer default: return nullptr; } - pos = end; + pos = static_cast(end - path.data()); + steps.emplace_back(pos, ptr); } return ptr; @@ -23257,7 +24623,7 @@ class json_pointer case detail::value_t::array: { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + if (JSON_HEDLEY_UNLIKELY(is_dash(reference_token))) { // "-" always fails the range check JSON_THROW(detail::out_of_range::create(detail::exception_id::pointer_past_the_end_index, detail::concat( @@ -23328,7 +24694,7 @@ class json_pointer case detail::value_t::array: { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + if (JSON_HEDLEY_UNLIKELY(is_dash(reference_token))) { // "-" cannot be used for const access JSON_THROW(detail::out_of_range::create(detail::exception_id::pointer_past_the_end_index, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr)); @@ -23390,7 +24756,7 @@ class json_pointer case detail::value_t::array: { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + if (JSON_HEDLEY_UNLIKELY(is_dash(reference_token))) { // "-" always fails the range check return nullptr; @@ -23467,7 +24833,7 @@ class json_pointer case detail::value_t::array: { - if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + if (JSON_HEDLEY_UNLIKELY(is_dash(reference_token))) { // "-" always fails the range check return false; @@ -23548,33 +24914,32 @@ class json_pointer } result.reserve(token_count); - // extract the reference tokens: - // - slash: position of the last read slash (or end of string) - // - start: position after the previous slash - for ( - // search for the first slash after the first character - std::size_t slash = reference_string.find_first_of('/', 1), - // set the beginning of the first reference token - start = 1; - // we can stop if start == 0 (if slash == string_t::npos) - start != 0; - // set the beginning of the next reference token - // (will eventually be 0 if slash == string_t::npos) - start = (slash == string_t::npos) ? 0 : slash + 1, - // find next slash - slash = reference_string.find_first_of('/', start)) + // extract the reference tokens: every one runs from after a slash to + // the next slash or the end of the string; a token without a '~' (the + // common case) is stored as it is, others are checked and unescaped + const char* const data = reference_string.data(); + const char* const end = data + size; + for (const char* start = data + 1;; ++start) { - // use the text between the beginning of the reference token - // (start) and the last slash (slash). - const auto count = (slash == string_t::npos ? reference_string.size() : slash) - start; - auto reference_token = string_t(reference_string.data() + start, count); - - // check reference tokens are properly escaped; a token without - // a '~' (the common case) needs neither check nor unescaping - std::size_t pos = reference_token.find_first_of('~'); - if (pos != string_t::npos) + const char* slash = start; + bool escaped = false; + while (slash != end && *slash != '/') { - for (; pos != string_t::npos; pos = reference_token.find_first_of('~', pos + 1)) + escaped = escaped || *slash == '~'; + ++slash; + } + + const auto count = static_cast(slash - start); + if (JSON_HEDLEY_LIKELY(!escaped)) + { + result.emplace_back(start, count); + } + else + { + auto reference_token = string_t(start, count); + + // check reference tokens are properly escaped + for (std::size_t pos = reference_token.find_first_of('~'); pos != string_t::npos; pos = reference_token.find_first_of('~', pos + 1)) { JSON_ASSERT(reference_token[pos] == '~'); @@ -23588,10 +24953,14 @@ class json_pointer } detail::unescape(reference_token); + result.push_back(std::move(reference_token)); } - // finally, store the reference token - result.push_back(std::move(reference_token)); + if (slash == end) + { + break; + } + start = slash; } return result; @@ -23626,6 +24995,52 @@ class json_pointer obj[key] = std::forward(v); } + /// the number of entries that flattening @a value adds: its primitive + /// values and empty containers + template + static std::size_t count_flattened(const BasicJsonType& value) + { + std::size_t count = 0; + std::vector pending(1, &value); + while (!pending.empty()) + { + const BasicJsonType* const v = pending.back(); + pending.pop_back(); + if (v->is_array() && !v->m_data.m_value.array->empty()) + { + for (const auto& element : *v->m_data.m_value.array) + { + pending.push_back(&element); + } + } + else if (v->is_object() && !v->m_data.m_value.object->empty()) + { + for (const auto& member : *v->m_data.m_value.object) + { + pending.push_back(&member.second); + } + } + else + { + ++count; + } + } + return count; + } + + // Reserve the storage for @a value of objects that have a reserve(), so + // that it does not have to be moved while it grows + template + static auto reserve_flattened(ObjectType& obj, const BasicJsonType& value, detail::priority_tag<1> /*unused*/) + -> decltype(obj.reserve(std::size_t {}), void()) + { + obj.reserve(count_flattened(value)); + } + + template + static void reserve_flattened(ObjectType& /*obj*/, const BasicJsonType& /*value*/, detail::priority_tag<0> /*unused*/) + {} + /// append the decimal representation of @a i to @a out static void append_index(string_t& out, std::size_t i) { @@ -23735,6 +25150,7 @@ class json_pointer } }; + reserve_flattened(object, value, detail::priority_tag<1> {}); enter(value); while (!stack.empty()) { @@ -23742,8 +25158,9 @@ class json_pointer // frame, which would invalidate a reference to it const BasicJsonType* const container = stack.back().container; - // drop the path of the previous child - path.resize(stack.back().path_length); + // the path of the previous child is replaced by the path of the + // next one; for an array element in one resize + const std::size_t path_length = stack.back().path_length; if (container->is_array()) { @@ -23757,8 +25174,20 @@ class json_pointer // iterate array and use index as a reference string ++stack.back().index; - path.push_back('/'); - append_index(path, i); + std::array digits{}; + std::size_t pos = digits.size(); + std::size_t rest = i; + do + { + digits[--pos] = static_cast('0' + (rest % 10)); + rest /= 10; + } + while (rest != 0); + const std::size_t count = digits.size() - pos; + path.resize(path_length + 1 + count); + char* const out = &path[path_length]; + out[0] = '/'; + std::memcpy(out + 1, digits.data() + pos, count); enter(array[i]); } else @@ -23772,6 +25201,7 @@ class json_pointer // iterate object and use keys as reference string ++stack.back().member; + path.resize(path_length); path.push_back('/'); detail::escape_into(path, it->first); enter(it->second); @@ -23867,8 +25297,28 @@ class json_pointer { end = key.size(); } + const std::size_t token_size = end - pos - 1; + + // a token that is a plain index of up to 9 digits, which is + // the case for almost all array elements, is not copied + std::size_t index = 0; + bool plain_index = current->type() == detail::value_t::array && token_size != 0 && token_size <= 9 && (token_size == 1 || data[pos + 1] != '0'); + for (std::size_t k = pos + 1; plain_index && k < end; ++k) + { + plain_index = data[k] >= '0' && data[k] <= '9'; + index = (index * 10) + static_cast(data[k] - '0'); + } + if (plain_index) + { + current = ¤t->operator[](static_cast(index)); + values.push_back(current); + ends.push_back(end); + pos = end; + continue; + } + token.clear(); - token.append(data + pos + 1, end - pos - 1); + token.append(data + pos + 1, token_size); if (token.find_first_of('~') != string_t::npos) { detail::unescape(token); @@ -23878,10 +25328,13 @@ class json_pointer { case detail::value_t::null: { - if (arrays.contains(data, pos)) + // a first child 0 makes the value an array without a look-up + if ((token_size == 1 && data[pos + 1] == '0') || arrays.contains(data, pos)) { // some reference token below this position is 0, so the // value is an array + *current = BasicJsonType(detail::value_t::array); + reserve_elements(*current->m_data.m_value.array, 2, detail::priority_tag<1> {}); current = ¤t->operator[](array_index(token)); } else @@ -24241,7 +25694,7 @@ NLOHMANN_JSON_NAMESPACE_END -#include // reverse +#include // max, min, reverse #include // array #include // isnan, isinf #include // uint8_t, uint16_t, uint32_t, uint64_t @@ -24321,6 +25774,66 @@ using output_adapter_t = std::shared_ptr>; /// output_vector_sink also appends writes of at least this size with insert() constexpr std::size_t max_reserve_size = 256; +/// @brief copy @a length characters, at most 32, without a call of memcpy() +/// +/// Short strings are the typical payload of a binary value, and a call of +/// memcpy() costs more than copying them. A copy of 16 to 32 characters is two +/// possibly overlapping 16-byte copies, one of 8 to 15 two 8-byte copies, and +/// so on down to a single character; each is a fixed-size load and store. +/// Nothing outside of [src, src + length) is read or written. +template +inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::true_type /*is_byte*/) noexcept +{ + JSON_ASSERT(length <= 32); + if (length >= 16) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 16); + std::memcpy(last.data(), src + (length - 16), 16); + std::memcpy(dst, first.data(), 16); + std::memcpy(dst + (length - 16), last.data(), 16); + } + else if (length >= 8) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 8); + std::memcpy(last.data(), src + (length - 8), 8); + std::memcpy(dst, first.data(), 8); + std::memcpy(dst + (length - 8), last.data(), 8); + } + else if (length >= 4) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 4); + std::memcpy(last.data(), src + (length - 4), 4); + std::memcpy(dst, first.data(), 4); + std::memcpy(dst + (length - 4), last.data(), 4); + } + else if (length != 0) + { + // 1 to 3: the first, the middle, and the last character + const CharType a = src[0]; + const CharType b = src[length >> 1]; + const CharType c = src[length - 1]; + dst[0] = a; + dst[length >> 1] = b; + dst[length - 1] = c; + } +} + +/// @brief copy @a length characters, of a character type wider than a byte +template +inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::false_type /*is_byte*/) noexcept +{ + std::memcpy(dst, src, length * sizeof(CharType)); +} + +/// the number of characters an empty vector is grown to by output_vector_sink +constexpr std::size_t min_vector_sink_capacity = 64; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -24344,7 +25857,7 @@ class output_vector_sink { public: explicit output_vector_sink(std::vector& vec) noexcept - : v(&vec), position(vec.size()) + : v(&vec), cursor(vec.data() + vec.size()), limit(cursor) {} output_vector_sink(const output_vector_sink&) = delete; @@ -24352,7 +25865,7 @@ class output_vector_sink output_vector_sink& operator=(output_vector_sink&&) = delete; output_vector_sink(output_vector_sink&& other) noexcept - : v(other.v), position(other.position) + : v(other.v), cursor(other.cursor), limit(other.limit) { other.v = nullptr; } @@ -24361,7 +25874,7 @@ class output_vector_sink { if (v != nullptr) { - v->resize(position); + v->resize(position()); } } @@ -24370,11 +25883,11 @@ class output_vector_sink JSON_HEDLEY_ALWAYS_INLINE void write_character(CharType c) { - if (JSON_HEDLEY_UNLIKELY(position == v->size())) + if (JSON_HEDLEY_UNLIKELY(cursor == limit)) { grow(1); } - (*v)[position++] = c; + *cursor++ = c; } // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null @@ -24383,7 +25896,7 @@ class output_vector_sink JSON_HEDLEY_ALWAYS_INLINE void write_characters(const CharType* s, std::size_t length) { - if (JSON_HEDLEY_UNLIKELY(v->size() - position < length)) + if (JSON_HEDLEY_UNLIKELY(space() < length)) { if (length >= max_reserve_size) { @@ -24392,10 +25905,15 @@ class output_vector_sink } grow(length); } - if (length != 0) + if (length <= 32) { - std::memcpy(v->data() + position, s, length * sizeof(CharType)); - position += length; + copy_short(cursor, s, length, std::integral_constant()); + cursor += length; + } + else + { + std::memcpy(cursor, s, length * sizeof(CharType)); + cursor += length; } } @@ -24410,21 +25928,37 @@ class output_vector_sink CharType* reserve(const std::size_t length) { JSON_ASSERT(length <= max_reserve_size); - if (JSON_HEDLEY_UNLIKELY(v->size() - position < length)) + if (JSON_HEDLEY_UNLIKELY(space() < length)) { grow(length); } - return v->data() + position; + return cursor; } /// keep the first @a length characters written to the space from reserve() JSON_HEDLEY_ALWAYS_INLINE void commit(const std::size_t length) noexcept { - position += length; + cursor += length; } + /// nothing to pass on: the characters are in the vector (see output_adapter_sink) + void flush() noexcept + {} + private: + /// number of characters of *v written so far + std::size_t position() const noexcept + { + return static_cast(cursor - v->data()); + } + + /// number of characters that fit the vector behind the write position + std::size_t space() const noexcept + { + return static_cast(limit - cursor); + } + /// make room for at least @a length more characters after the position; /// kept out of line so that the write functions stay small enough for /// GCC to inline them into the writer @@ -24432,12 +25966,15 @@ class output_vector_sink void grow(const std::size_t length) { // use the capacity the vector already has, else at least double it; - // never less than 64 characters, so that a small value does not + // never less than min_vector_sink_capacity, so that a small value does not // reallocate for every few bytes - const std::size_t needed = position + length; - v->resize((std::max)(std::size_t{64}, needed <= v->capacity() + const std::size_t written = position(); + const std::size_t needed = written + length; + v->resize((std::max)(min_vector_sink_capacity, needed <= v->capacity() ? v->capacity() : (std::max)(needed, v->capacity() * 2))); + cursor = v->data() + written; + limit = v->data() + v->size(); } /// append @a length (at least max_reserve_size) characters that do not fit @@ -24447,20 +25984,24 @@ class output_vector_sink JSON_HEDLEY_NEVER_INLINE void append(const CharType* s, const std::size_t length) { - v->resize(position); - if (v->capacity() - position < length) + const std::size_t written = position(); + v->resize(written); + if (v->capacity() - written < length) { // grow geometrically, with room for the writes that follow, so // that the next grow() need not reallocate right away - v->reserve((std::max)(v->capacity() * 2, position + length + max_reserve_size)); + v->reserve((std::max)(v->capacity() * 2, written + length + max_reserve_size)); } v->insert(v->end(), s, s + length); - position = v->size(); + cursor = v->data() + v->size(); + limit = cursor; } std::vector* v; - /// number of characters of *v written so far - std::size_t position; + /// the write position: in *v, at the number of characters written so far + CharType* cursor; + /// the end of *v + CharType* limit; }; /// output adapter for byte vectors @@ -24543,9 +26084,17 @@ class output_string_adapter : public output_adapter_protocol /// /// Wraps the polymorphic output_adapter_t so the same binary_writer template can /// also target arbitrary adapters (output streams, strings, user-provided -/// adapters) via the `output_adapter`-based overloads. Each write still goes -/// through one virtual call, exactly as before; only the concrete sinks above -/// avoid it. +/// adapters) via the `output_adapter`-based overloads. +/// +/// A write through the adapter is a virtual call, and for a stream a call with +/// a sentry on top, which costs more than encoding the few bytes of most values. +/// The sink therefore collects what is written in a buffer and passes it on in +/// one piece when it is full, a write is too large for it, or @ref flush is +/// called. Everything written reaches the adapter in the same order and with +/// the same bytes; the destructor passes on what is left, so that a write that +/// throws leaves the adapter with exactly the bytes written before it. Only +/// @ref flush can report an error of the adapter (the destructor must not +/// throw), so it has to be called once the value is written. template class output_adapter_sink { @@ -24556,36 +26105,121 @@ class output_adapter_sink JSON_ASSERT(oa); } + output_adapter_sink(const output_adapter_sink&) = delete; + output_adapter_sink& operator=(const output_adapter_sink&) = delete; + output_adapter_sink& operator=(output_adapter_sink&&) = delete; + + output_adapter_sink(output_adapter_sink&& other) noexcept + : oa(std::move(other.oa)), used(other.used) + { + std::memcpy(buffer.data(), other.buffer.data(), other.used * sizeof(CharType)); + other.used = 0; + } + + ~output_adapter_sink() + { + // a normal return has flushed already, so this is for a write that threw + if (used != 0) + { + JSON_TRY + { + flush(); + } + JSON_CATCH(...) + { + // nothing can be done about an adapter that throws here + } + } + } + + JSON_HEDLEY_ALWAYS_INLINE void write_character(CharType c) { - oa->write_character(c); + if (JSON_HEDLEY_UNLIKELY(used == buffer_size)) + { + flush(); + } + buffer[used++] = c; } // no JSON_HEDLEY_NON_NULL: forwards (null, 0) for empty payloads, exactly as // the type-erased path already did before this sink existed + JSON_HEDLEY_ALWAYS_INLINE void write_characters(const CharType* s, std::size_t length) { - oa->write_characters(s, length); + if (JSON_HEDLEY_UNLIKELY(buffer_size - used < length)) + { + write_large(s, length); + return; + } + if (length <= 32) + { + copy_short(buffer.data() + used, s, length, std::integral_constant()); + used += length; + } + else + { + std::memcpy(buffer.data() + used, s, length * sizeof(CharType)); + used += length; + } } /// space for writing up to @a length characters (see /// output_vector_sink::reserve) - CharType* reserve(const std::size_t length) noexcept + JSON_HEDLEY_ALWAYS_INLINE + CharType* reserve(const std::size_t length) { JSON_ASSERT(length <= max_reserve_size); - static_cast(length); - return reserved.data(); + if (JSON_HEDLEY_UNLIKELY(buffer_size - used < length)) + { + flush(); + } + return buffer.data() + used; } - /// write the first @a length characters written to the space from reserve() - void commit(const std::size_t length) + /// keep the first @a length characters written to the space from reserve() + JSON_HEDLEY_ALWAYS_INLINE + void commit(const std::size_t length) noexcept { - oa->write_characters(reserved.data(), length); + used += length; + } + + /// pass everything written so far on to the adapter + void flush() + { + if (used != 0) + { + // reset first: the characters count as written even if the adapter throws + const std::size_t length = used; + used = 0; + oa->write_characters(buffer.data(), length); + } } private: + /// a write that does not fit the space left in the buffer + JSON_HEDLEY_NEVER_INLINE + void write_large(const CharType* s, const std::size_t length) + { + flush(); + if (length >= buffer_size) + { + oa->write_characters(s, length); + } + else + { + std::memcpy(buffer.data(), s, length * sizeof(CharType)); + used = length; + } + } + + /// large enough for reserve() at any time + static constexpr std::size_t buffer_size = 4 * max_reserve_size; + output_adapter_t oa; - std::array reserved{}; + /// number of characters of @ref buffer not yet passed on + std::size_t used = 0; + std::array buffer{}; }; /// @brief whether std::basic_string has a non-deprecated std::char_traits @@ -24744,6 +26378,21 @@ std::size_t binary_reserve_hint(const BasicJsonType& j) return 0; } +/*! +@brief the capacity to reserve for the result of a binary serialization into a + std::vector + +The hint of @ref binary_reserve_hint, but not less than the 64 characters that +output_vector_sink grows an empty vector to on its first write anyway. A small +document then costs one allocation, instead of an allocation of a few bytes that +the first write replaces with one of 64. +*/ +template +std::size_t binary_initial_capacity(const BasicJsonType& j) +{ + return (std::max)(binary_reserve_hint(j), min_vector_sink_capacity); +} + /*! @brief serialization to BJData, BON8, BSON, CBOR, MessagePack, and UBJSON values */ @@ -24792,9 +26441,22 @@ class binary_writer template < typename SinkType = OutputSinkType, typename std::enable_if < std::is_constructible>::value, int >::type = 0 > explicit binary_writer(output_adapter_t adapter, const error_handler_t error_handler_ = binary_writer_default_error_handler()) - : oa(SinkType(std::move(adapter))), error_handler(error_handler_) + : oa(std::move(adapter)), error_handler(error_handler_) {} + /*! + @brief pass on everything written so far to the output adapter + + A writer over an output_adapter_sink buffers what it writes; this has to + be called after the last write, or the buffered end is only passed on, and + an error of the adapter lost, when the writer is destroyed. A writer over + an output_vector_sink writes straight into the vector. + */ + void flush() + { + oa.flush(); + } + /*! @param[in] j JSON value to serialize @throw type_error.316 if a string value or an object key is not valid @@ -24961,16 +26623,7 @@ class binary_writer case value_t::object: { - // step 1: write control byte and the object size - write_cbor_head(0xA0, j.m_data.m_value.object->size()); - - for (const auto& el : *j.m_data.m_value.object) - { - // el.first is written directly (not via a temporary - // basic_json), with the object as diagnostics context - write_cbor_key(el.first, j); - write_cbor(el.second, depth + 1); - } + write_cbor_object(j, depth); break; } @@ -24990,7 +26643,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(length))) { - JSON_THROW(out_of_range::create(exception_id::length_too_large, concat("MessagePack length ", std::to_string(length), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), &j)); + throw_msgpack_length_too_large(length, j); } static_cast(j); @@ -25079,7 +26732,7 @@ class binary_writer case value_t::number_float: { - write_compact_float(j.m_data.m_value.number_float, to_char_type(0xCA), to_char_type(0xCB)); + write_msgpack_float(j.m_data.m_value.number_float); break; } @@ -25169,7 +26822,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(j.m_data.m_value.binary->subtype() > (std::numeric_limits::max)())) { - JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(j.m_data.m_value.binary->subtype()), " is too large for the MessagePack ext type (max 255)"), &j)); + throw_subtype_out_of_range(j.m_data.m_value.binary->subtype(), "the MessagePack ext type", j); } write_number(static_cast(j.m_data.m_value.binary->subtype())); @@ -25186,15 +26839,7 @@ class binary_writer case value_t::object: { // step 1: write control byte and the object size - write_msgpack_object_prefix(j.m_data.m_value.object->size(), j); - - for (const auto& el : *j.m_data.m_value.object) - { - // as in write_cbor, el.first is written directly with the - // object as diagnostics context - write_msgpack_key(el.first, j); - write_msgpack(el.second, depth + 1); - } + write_msgpack_object(j, depth); break; } @@ -25256,14 +26901,16 @@ class binary_writer } case value_t::number_integer: - { - write_number_with_ubjson_prefix(j.m_data.m_value.number_integer, add_prefix, use_bjdata); - break; - } - case value_t::number_unsigned: { - write_number_with_ubjson_prefix(j.m_data.m_value.number_unsigned, add_prefix, use_bjdata); + if (j.m_data.m_type == value_t::number_integer) + { + write_number_with_ubjson_prefix(j.m_data.m_value.number_integer, add_prefix, use_bjdata); + } + else + { + write_number_with_ubjson_prefix(j.m_data.m_value.number_unsigned, add_prefix, use_bjdata); + } break; } @@ -25275,17 +26922,7 @@ class binary_writer case value_t::string: { - string_t storage; - const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage); - - if (add_prefix) - { - oa.write_character(to_char_type('S')); - } - write_number_with_ubjson_prefix(value.size(), true, use_bjdata); - oa.write_characters( - reinterpret_cast(value.data()), - value.size()); + write_ubjson_text(*j.m_data.m_value.string, j, add_prefix, use_bjdata); break; } @@ -25294,9 +26931,19 @@ class binary_writer bool prefix_required = true; const bool write_closer = write_ubjson_start_array(j, use_count, use_type, add_prefix, use_bjdata, prefix_required); - for (const auto& el : *j.m_data.m_value.array) { - write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + const auto& array = *j.m_data.m_value.array; + if (starts_with_ubjson_number(array, use_bjdata, std::integral_constant())) + { + write_ubjson_array_elements(array, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth); + } + else + { + for (const auto& el : array) + { + write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + } + } } if (write_closer) @@ -25318,7 +26965,7 @@ class binary_writer { if (!use_count) { - JSON_THROW(other_error::create(exception_id::size_marker_required, "use_type requires use_size = true", &j)); + throw_size_marker_required(j); } oa.write_character(to_char_type('$')); oa.write_character(bjdata_draft3 ? 'B' : 'U'); @@ -25338,13 +26985,7 @@ class binary_writer } else { - for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) - { - oa.write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); - // the cast is needed for binary types whose value type - // is not an integer (e.g., std::byte) - oa.write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); - } + write_ubjson_binary_bytes(*j.m_data.m_value.binary, bjdata_draft3); } if (!use_count) @@ -25368,18 +27009,7 @@ class binary_writer bool prefix_required = true; const bool write_closer = write_ubjson_start_object(j, use_count, use_type, add_prefix, use_bjdata, prefix_required); - for (const auto& el : *j.m_data.m_value.object) - { - // a converted key must outlive the reference returned by sanitize_utf8_for_write - const object_key_string_t key_string = el.first; - string_t storage; - const string_t& key = sanitize_utf8_for_write(key_string, j, storage); - write_number_with_ubjson_prefix(key.size(), true, use_bjdata); - oa.write_characters( - reinterpret_cast(key.data()), - key.size()); - write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); - } + write_ubjson_members(j, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth); if (write_closer) { @@ -25544,7 +27174,8 @@ class binary_writer @brief throws because the integer @a j is too large for @a format_name @throw out_of_range.407 always */ - JSON_HEDLEY_NO_RETURN static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) { // neither std::to_string(), which has no overload for integer types // wider than 64 bits, nor dump(), which would also instantiate the @@ -25560,13 +27191,53 @@ class binary_writer JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", number, " cannot be represented by ", format_name, " as it does not fit ", range), &j)); } + // The exceptions below are thrown from functions of their own, out of line: + // the code that builds a message (concat, std::to_string) would otherwise + // make the frame of the function that checks, which most calls never throw. + + /// @throw out_of_range.412 always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_msgpack_length_too_large(const std::size_t length, const BasicJsonType& j) + { + static_cast(length); // unused when JSON_NOEXCEPTION is defined + static_cast(j); + JSON_THROW(out_of_range::create(exception_id::length_too_large, concat("MessagePack length ", std::to_string(length), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), &j)); + } + + /// @throw out_of_range.412 always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_bson_length_too_large(const std::size_t size) + { + static_cast(size); // unused when JSON_NOEXCEPTION is defined + JSON_THROW(out_of_range::create(exception_id::length_too_large, concat("BSON length ", std::to_string(size), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), nullptr)); + } + + /// @throw out_of_range.415 always + template + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_subtype_out_of_range(const SubtypeType subtype, const char* what, const BasicJsonType& j) + { + static_cast(subtype); // unused when JSON_NOEXCEPTION is defined + static_cast(what); + static_cast(j); + JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(subtype), " is too large for ", what, " (max 255)"), &j)); + } + + /// @throw other_error.407 always + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_size_marker_required(const BasicJsonType& j) + { + static_cast(j); // unused when JSON_NOEXCEPTION is defined + JSON_THROW(other_error::create(exception_id::size_marker_required, "use_type requires use_size = true", &j)); + } + void write_msgpack_array_prefix(const std::size_t N, const BasicJsonType& j) { const auto n = to_msgpack_length(N, j); if (n <= 15) { // fixarray - write_number(static_cast(0x90 | n)); + oa.write_character(to_char_type(static_cast(0x90 | n))); } else if (n <= (std::numeric_limits::max)()) { @@ -25588,7 +27259,7 @@ class binary_writer if (n <= 15) { // fixmap - write_number(static_cast(0x80 | (n & 0xF))); + oa.write_character(to_char_type(static_cast(0x80 | (n & 0xF)))); } else if (n <= (std::numeric_limits::max)()) { @@ -25817,7 +27488,7 @@ class binary_writer { if (!use_count) { - JSON_THROW(other_error::create(exception_id::size_marker_required, "use_type requires use_size = true", &j)); + throw_size_marker_required(j); } const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); const bool same_prefix = std::all_of(j.begin() + 1, j.end(), @@ -25846,8 +27517,7 @@ class binary_writer if (use_count) { - oa.write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); + write_ubjson_count(j.m_data.m_value.array->size(), use_bjdata); } return !use_count; @@ -25867,7 +27537,7 @@ class binary_writer { if (!use_count) { - JSON_THROW(other_error::create(exception_id::size_marker_required, "use_type requires use_size = true", &j)); + throw_size_marker_required(j); } const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); const bool same_prefix = std::all_of(j.begin(), j.end(), @@ -25886,8 +27556,7 @@ class binary_writer if (use_count) { - oa.write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); + write_ubjson_count(j.m_data.m_value.object->size(), use_bjdata); } return !use_count; @@ -26053,12 +27722,7 @@ class binary_writer // a converted key must outlive the reference returned by sanitize_utf8_for_write const object_key_string_t key_string = current.object_it->first; - string_t storage; - const string_t& key = sanitize_utf8_for_write(key_string, j, storage); - write_number_with_ubjson_prefix(key.size(), true, use_bjdata); - oa.write_characters( - reinterpret_cast(key.data()), - key.size()); + write_ubjson_text(key_string, j, false, use_bjdata); const BasicJsonType* child = &(current.object_it->second); const bool child_prefix = current.prefix_required; ++stack.back().object_it; @@ -26081,18 +27745,85 @@ class binary_writer */ std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) { - const auto it = name.find(static_cast(0)); + const std::size_t it = find_nul(name); if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos)) { - JSON_THROW(out_of_range::create(exception_id::bson_key_with_null, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j)); + throw_bson_key_with_null(it, j); } + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + return calc_bson_checked_entry_header_size(name, j); + } + + /// @brief throw out_of_range.409 for a name with U+0000 at byte @a index; + /// out of line, so that the message does not burden the check + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_bson_key_with_null(const std::size_t index, const BasicJsonType& j) + { + static_cast(index); // only used when exceptions are enabled + static_cast(j); + JSON_THROW(out_of_range::create(exception_id::bson_key_with_null, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(index), ")"), &j)); + } + + /// @brief the size of a BSON document entry header for a name that the + /// error handler checks or sanitizes; out of line, as the string_t + /// for the sanitized copy would otherwise burden every call of + /// @ref calc_bson_entry_header_size + JSON_HEDLEY_NEVER_INLINE + std::size_t calc_bson_checked_entry_header_size(const string_t& name, const BasicJsonType& j) + { string_t storage; const string_t& sanitized = sanitize_utf8_for_write(name, j, storage); return /*id*/ 1ul + sanitized.size() + /*zero-terminator*/1u; } + /*! + @brief the position of the first U+0000 in @a name, or string_t::npos + + The same as `name.find(0)`, without the calls into the standard library for + names of a few characters: it looks at eight characters at a time. + */ + static std::size_t find_nul(const string_t& name) noexcept + { + return find_nul(name, std::integral_constant()); + } + + static std::size_t find_nul(const string_t& name, std::true_type /*is_byte*/) noexcept + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + constexpr std::uint64_t high = 0x8080808080808080ull; + const std::size_t size = name.size(); + const char* const data = reinterpret_cast(name.data()); + std::size_t pos = 0; + for (; pos + 8 <= size; pos += 8) + { + std::uint64_t word = 0; + std::memcpy(&word, data + pos, sizeof(word)); + if (((word - ones) & ~word & high) != 0) + { + break; + } + } + for (; pos < size; ++pos) + { + if (data[pos] == '\0') + { + return pos; + } + } + return BasicJsonType::string_t::npos; + } + + static std::size_t find_nul(const string_t& name, std::false_type /*is_byte*/) noexcept + { + return name.find(static_cast(0)); + } + /*! @brief Checks that @a size fits into the 32-bit length field used by BSON @return The size as a signed 32-bit integer @@ -26102,7 +27833,7 @@ class binary_writer { if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) { - JSON_THROW(out_of_range::create(exception_id::length_too_large, concat("BSON length ", std::to_string(size), " exceeds maximum of ", std::to_string((std::numeric_limits::max)())), nullptr)); + throw_bson_length_too_large(size); } return static_cast(size); @@ -26172,13 +27903,25 @@ class binary_writer { if (JSON_HEDLEY_LIKELY(value_in_range_of(value.size()))) { - string_t storage; - const string_t& sanitized = sanitize_utf8_for_write(value, j, storage); - return sizeof(std::int32_t) + sanitized.size() + 1ul; + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + return calc_bson_checked_string_size(value, j); } return sizeof(std::int32_t) + value.size() + 1ul; } + /// @brief the size of a BSON string that the error handler checks or + /// sanitizes; out of line, like @ref calc_bson_checked_entry_header_size + JSON_HEDLEY_NEVER_INLINE + std::size_t calc_bson_checked_string_size(const string_t& value, const BasicJsonType& j) + { + string_t storage; + const string_t& sanitized = sanitize_utf8_for_write(value, j, storage); + return sizeof(std::int32_t) + sanitized.size() + 1ul; + } + /*! @brief Writes a BSON element with key @a name and string value @a value @@ -26288,7 +28031,7 @@ class binary_writer if (value.has_subtype() && JSON_HEDLEY_UNLIKELY(value.subtype() > (std::numeric_limits::max)())) { - JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(value.subtype()), " is too large for the BSON binary subtype (max 255)"), &j)); + throw_subtype_out_of_range(value.subtype(), "the BSON binary subtype", j); } return sizeof(std::int32_t) + value.size() + 1ul; @@ -26368,12 +28111,198 @@ class binary_writer } } + /// the longest name for which write_bson_scalar() writes an entry through a + /// single reservation: the type, name, terminating null byte, and the length + /// of a string or binary value (4 bytes) or the largest fixed-size value (8 + /// bytes, a double or a 64-bit integer) must fit max_reserve_size + static constexpr std::size_t bson_scalar_max_name_size = max_reserve_size - 2 - 8 - 1; + + /*! + @brief store @a n at @a out in little-endian byte order + */ + template + void store_little_endian(CharType* out, const NumberType n) const noexcept + { + std::memcpy(out, &n, sizeof(NumberType)); + if (!is_little_endian) + { + std::reverse(out, out + sizeof(NumberType)); + } + } + + /*! + @brief encode the type and the value of the BSON document entry for @a j + if it has a fixed size + @param[in] out where the type byte goes + @param[in] value_out where the value goes: the characters after the name + and its terminating null byte + @return the end of the entry, or nullptr (and nothing is written to + @a out) if @a j is not a boolean, number, or null; at most 9 + characters are written to @a out and @a value_out + + The entry is encoded as @ref write_bson_value does. + */ + CharType* encode_bson_fixed_entry(CharType* out, CharType* value_out, const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::boolean: + out[0] = to_char_type(0x08); + *value_out++ = j.m_data.m_value.boolean ? to_char_type(0x01) : to_char_type(0x00); + return value_out; + + case value_t::number_float: + out[0] = to_char_type(0x01); + store_little_endian(value_out, static_cast(j.m_data.m_value.number_float)); + return value_out + sizeof(double); + + case value_t::number_integer: + { + const auto value = static_cast(j.m_data.m_value.number_integer); + if ((std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)()) + { + out[0] = to_char_type(0x10); // int32 + store_little_endian(value_out, static_cast(value)); + return value_out + sizeof(std::int32_t); + } + out[0] = to_char_type(0x12); // int64 + store_little_endian(value_out, value); + return value_out + sizeof(std::int64_t); + } + + case value_t::number_unsigned: + { + const auto value = conditional_static_cast(j.m_data.m_value.number_unsigned); + if (value <= static_cast((std::numeric_limits::max)())) + { + out[0] = to_char_type(0x10); // int32 + store_little_endian(value_out, static_cast(value)); + return value_out + sizeof(std::int32_t); + } + if (value <= static_cast((std::numeric_limits::max)())) + { + out[0] = to_char_type(0x12); // int64 + store_little_endian(value_out, static_cast(value)); + return value_out + sizeof(std::int64_t); + } + out[0] = to_char_type(0x11); // uint64 + store_little_endian(value_out, value); + return value_out + sizeof(std::uint64_t); + } + + case value_t::null: + out[0] = to_char_type(0x0A); + return value_out; + + case value_t::string: + case value_t::binary: + case value_t::object: + case value_t::array: + case value_t::discarded: + default: + return nullptr; + } + } + + /*! + @brief Writes the BSON document entry with key @a name for @a j, which is + neither an object nor an array, with the key and a fixed-size value + (or the header of a string or binary value) in one reservation + @return whether the entry was written; false for a discarded value, a + string or binary value whose length does not fit the length + field, in which case nothing was written + @pre @a name is at most bson_scalar_max_name_size characters long and is + not changed by the error handler + */ + bool write_bson_scalar(const string_t& name, const BasicJsonType& j) + { + const std::size_t name_size = name.size(); + // a short string is written whole: its type, the name and its null + // byte, the length, the characters, and their null byte + const bool whole_string = j.is_string() && j.m_data.m_value.string->size() <= max_reserve_size - name_size - 7; + CharType* const out = oa.reserve(whole_string ? name_size + j.m_data.m_value.string->size() + 7 : name_size + 2 + 8); + // the name and its terminating null byte follow the type byte + CharType* p = out + 1; + copy_characters(p, reinterpret_cast(name.data()), name_size); + p += name_size; + *p++ = to_char_type(0x00); + + if (const CharType* const end = encode_bson_fixed_entry(out, p, j)) + { + oa.commit(static_cast(end - out)); + return true; + } + + if (whole_string) + { + const string_t& value = *j.m_data.m_value.string; + out[0] = to_char_type(0x02); + store_little_endian(p, static_cast(value.size() + 1ul)); + p += sizeof(std::int32_t); + copy_characters(p, reinterpret_cast(value.data()), value.size()); + p += value.size(); + // the terminating null byte is written explicitly rather than + // taken from the buffer, so that string_t::data() need not be + // null-terminated + *p++ = to_char_type(0x00); + oa.commit(static_cast(p - out)); + return true; + } + + if (j.is_string()) + { + const string_t& value = *j.m_data.m_value.string; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(value.size() + 1ul))) + { + return false; + } + out[0] = to_char_type(0x02); + store_little_endian(p, static_cast(value.size() + 1ul)); + p += sizeof(std::int32_t); + oa.commit(static_cast(p - out)); + // the terminating null byte is written explicitly rather than + // taken from the buffer, so that string_t::data() need not be + // null-terminated + oa.write_characters(reinterpret_cast(value.data()), value.size()); + oa.write_character(to_char_type(0x00)); + return true; + } + + if (j.is_binary()) + { + const binary_t& value = *j.m_data.m_value.binary; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(value.size()))) + { + return false; + } + out[0] = to_char_type(0x05); + store_little_endian(p, static_cast(value.size())); + p += sizeof(std::int32_t); + *p++ = value.has_subtype() ? to_char_type(static_cast(value.subtype())) : to_char_type(0x00); + oa.commit(static_cast(p - out)); + oa.write_characters(reinterpret_cast(value.data()), value.size()); + return true; + } + + return false; + } + /*! @brief Writes the BSON document entry with key @a name for @a j, which is neither an object nor an array */ void write_bson_value(const string_t& name, const BasicJsonType& j) { + // The strings are written as they are, as they are neither checked nor + // sanitized again here by these two error handlers. The name has to leave + // room for the larger payloads (see write_bson_scalar). + if (name.size() <= bson_scalar_max_name_size + && (error_handler == error_handler_t::keep || error_handler == error_handler_t::strict) + && write_bson_scalar(name, j)) + { + return; + } + switch (j.type()) { case value_t::binary: @@ -26426,6 +28355,9 @@ class binary_writer write_number(to_bson_length(size), true); } + /// space for the name of an array element, the decimal index, in the frame + static constexpr std::size_t bson_index_name_space = 24; + /// @brief an object or array of the BSON document being sized or written struct bson_frame { @@ -26439,28 +28371,196 @@ class binary_writer } } + /// @ref calc_bson_sizes only: make @ref index the next one and count the + /// digits of its name + void count_index() noexcept + { + if (++index == index_boundary) + { + ++index_name_size; + index_boundary *= 10; + } + } + + /// @ref write_bson_document only: make @ref index the next one and + /// update its decimal name + void next_index() noexcept + { + ++index; + std::size_t i = index_name_size; + while (i > 0 && index_name[i - 1] == '9') + { + index_name[--i] = '0'; + } + if (i > 0) + { + ++index_name[i - 1]; + } + else + { + // all digits were 9: 10, 100, ... + index_name[0] = '1'; + index_name[index_name_size++] = '0'; + } + } + /// the object or array const BasicJsonType* value; /// objects: the next member typename BasicJsonType::object_t::const_iterator member{}; /// arrays: the index of the next element std::size_t index = 0; + /// arrays: the name BSON gives that element, its decimal index (at most + /// 2^64 - 1, which has 20 digits), in space for copying a few characters + /// more; only maintained by @ref write_bson_document + std::array index_name{{'0'}}; + /// arrays: the number of digits of the index + std::size_t index_name_size = 1; + /// arrays: the first index with one more digit; only maintained by + /// @ref calc_bson_sizes + std::uint64_t index_boundary = 10; /// @ref calc_bson_sizes only: where its size goes in the table std::size_t size_slot; /// @ref calc_bson_sizes only: the size of its entries seen so far std::size_t entries_size = 0; }; - /*! - @brief creates the name BSON gives the array element with index @a index - @param[out] name receives the decimal index - */ - static void create_bson_index_name(const std::size_t index, string_t& name) + /// whether the BSON entry of @a j has a fixed size, which is the case for + /// a boolean, a number, and null + static bool is_bson_fixed_size(const BasicJsonType& j) noexcept { - // the index is built as a std::string; convert explicitly, as the - // two are only implicitly convertible for some string types - const auto key = std::to_string(index); - name = string_t(key.data(), key.size()); + return j.is_number() || j.is_boolean() || j.is_null(); + } + + /*! + @brief Writes the entries of the array in @a frame that have a fixed size + (booleans, numbers, and null), from its next element on, until it + reaches another element or the end of the array + + The index of the element and its name are advanced in @a frame. + + The entries are encoded straight into space reserved from the sink for as + many of them as fit, with the index, its digits, and the write pointer + kept in local variables: stores of characters may alias the frame, and + would force them to be loaded from it again for every element. + */ + void write_bson_array_fixed_entries(bson_frame& frame) + { + const auto& array = *frame.value->m_data.m_value.array; + const std::size_t count = array.size(); + std::size_t index = frame.index; + std::size_t name_size = frame.index_name_size; + std::array name = frame.index_name; + + // an entry is at most the type, 20 digits, the null byte, and 8 bytes + constexpr std::size_t max_entry = 1 + 20 + 1 + 8; + while (index < count) + { + CharType* const start = oa.reserve(max_reserve_size); + CharType* out = start; + const CharType* const limit = start + (max_reserve_size - max_entry); + while (out <= limit && index < count) + { + CharType* p = out + 1; + std::memcpy(p, name.data(), name.size()); + p += name_size; + *p++ = to_char_type(0x00); + CharType* const end = encode_bson_fixed_entry(out, p, array[index]); + if (end == nullptr) + { + break; + } + out = end; + + ++index; + std::size_t i = name_size; + while (i > 0 && name[i - 1] == '9') + { + name[--i] = '0'; + } + if (i > 0) + { + ++name[i - 1]; + } + else + { + // all digits were 9: 10, 100, ... + name[0] = '1'; + name[name_size++] = '0'; + } + } + oa.commit(static_cast(out - start)); + if (out <= limit && index < count) + { + // stopped at an element that has no fixed size + break; + } + } + + frame.index = index; + frame.index_name_size = name_size; + frame.index_name = name; + } + + /*! + @brief Calculates the size of the BSON array @a j if all its elements are + booleans, numbers, or null + @param[out] size the size of the array, including its own size field + @return false (and nothing is known about @a size) if an element is not, + or is an integer that does not fit 64 bits (which the generic + path rejects), which is not looked at further + */ + bool calc_bson_fixed_array_size(const BasicJsonType& j, std::size_t& size) const noexcept + { + std::size_t total = sizeof(std::int32_t) + 1ul; + std::size_t index_name_size = 1; + std::uint64_t index_boundary = 10; + std::uint64_t index = 0; + for (const auto& el : *j.m_data.m_value.array) + { + // the type, the decimal index, and the terminating null byte + total += 2ul + index_name_size; + if (++index == index_boundary) + { + ++index_name_size; + index_boundary *= 10; + } + switch (el.type()) + { + case value_t::boolean: + total += 1ul; + break; + case value_t::number_float: + total += 8ul; + break; + case value_t::number_integer: + // an integer beyond 64 bits is left to the generic path, which throws + if (JSON_HEDLEY_UNLIKELY(!integer_fits(el.m_data.m_value.number_integer))) + { + return false; + } + total += calc_bson_integer_size(static_cast(el.m_data.m_value.number_integer)); + break; + case value_t::number_unsigned: + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(el.m_data.m_value.number_unsigned))) + { + return false; + } + total += calc_bson_unsigned_size(conditional_static_cast(el.m_data.m_value.number_unsigned)); + break; + case value_t::null: + break; + case value_t::string: + case value_t::binary: + case value_t::object: + case value_t::array: + case value_t::discarded: + default: + return false; + } + } + size = total; + return true; } /*! @@ -26494,8 +28594,6 @@ class binary_writer // is in; nothing is allocated unless the document nests bson_frame current(&document); std::vector parents; - // string_t need not be default constructible - string_t index_name("", 0); while (true) { @@ -26526,9 +28624,10 @@ class binary_writer while (nested == nullptr && current.index < array.size()) { const BasicJsonType& el = array[current.index]; - create_bson_index_name(current.index, index_name); - current.entries_size += calc_bson_entry_header_size(index_name, el); - ++current.index; + // the type, the decimal index (which is valid UTF-8 and + // has no U+0000), and the terminating null byte + current.entries_size += 2ul + current.index_name_size; + current.count_index(); if (el.is_structured()) { nested = ⪙ @@ -26542,6 +28641,15 @@ class binary_writer if (nested != nullptr) { + // an array of booleans, numbers and null: sized without a frame + std::size_t fixed_size = 0; + if (nested->is_array() && calc_bson_fixed_array_size(*nested, fixed_size)) + { + nested_sizes.push_back(fixed_size); + current.entries_size += fixed_size; + continue; + } + // its size is added to the current one's once it is done nested_sizes.push_back(0); parents.push_back(std::move(current)); @@ -26583,8 +28691,6 @@ class binary_writer bson_frame current(&document); std::vector parents; std::size_t next_size = 0; - // string_t need not be default constructible - string_t index_name("", 0); while (true) { @@ -26614,9 +28720,19 @@ class binary_writer const auto& array = *current.value->m_data.m_value.array; while (nested == nullptr && current.index < array.size()) { + if (is_bson_fixed_size(array[current.index])) + { + write_bson_array_fixed_entries(current); + if (current.index == array.size()) + { + break; + } + } + + // a string, binary value, object or array const BasicJsonType& el = array[current.index]; - create_bson_index_name(current.index, index_name); - ++current.index; + const string_t index_name(current.index_name.data(), current.index_name_size); + current.next_index(); if (el.is_structured()) { write_bson_nested_head(index_name, el, nested_sizes[next_size++]); @@ -26631,6 +28747,20 @@ class binary_writer if (nested != nullptr) { + if (nested->is_array()) + { + // an array of booleans, numbers and null is written without a frame on the stack + bson_frame leaf(nested); + write_bson_array_fixed_entries(leaf); + if (leaf.index == nested->m_data.m_value.array->size()) + { + oa.write_character(to_char_type(0x00)); + continue; + } + parents.push_back(std::move(current)); + current = std::move(leaf); + continue; + } parents.push_back(std::move(current)); current = bson_frame(nested); continue; @@ -26793,6 +28923,68 @@ class binary_writer return encode_first_byte_and_tail(out, first_byte, size_class, argument); } + /// write the head of an array or object: one byte for a size of up to 23, else as + /// @ref write_cbor_head + void write_cbor_container_head(const std::uint8_t major_type, const std::uint64_t size) + { + if (JSON_HEDLEY_LIKELY(size <= 0x17)) + { + oa.write_character(to_char_type(static_cast(major_type + size))); + return; + } + write_cbor_head(major_type, size); + } + + /*! + @brief write an object value (head and members) as CBOR + + A member whose value is null, a boolean, an integer, or a string is written + without a call of @ref write_cbor. Kept out of line, like @ref write_cbor_array. + */ + JSON_HEDLEY_NEVER_INLINE + void write_cbor_object(const BasicJsonType& j, const std::size_t depth) + { + // step 1: write control byte and the object size + write_cbor_container_head(0xA0, j.m_data.m_value.object->size()); + + for (const auto& el : *j.m_data.m_value.object) + { + // el.first is written directly (not via a temporary + // basic_json), with the object as diagnostics context + write_cbor_key(el.first, j); + + const BasicJsonType& value = el.second; + switch (value.type()) + { + case value_t::null: + oa.write_character(to_char_type(0xF6)); + break; + + case value_t::boolean: + oa.write_character(value.m_data.m_value.boolean ? to_char_type(0xF5) : to_char_type(0xF4)); + break; + + case value_t::number_integer: + case value_t::number_unsigned: + write_cbor_integer(value, std::integral_constant()); + break; + + case value_t::string: + write_cbor_string(*value.m_data.m_value.string, value); + break; + + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::number_float: + case value_t::discarded: + default: + write_cbor(value, depth + 1); + break; + } + } + } + /*! @brief write an array value (head and elements) as CBOR @@ -26804,15 +28996,22 @@ class binary_writer JSON_HEDLEY_NEVER_INLINE void write_cbor_array(const BasicJsonType& j, const std::size_t depth) { - write_cbor_head(0x80, j.m_data.m_value.array->size()); - write_cbor_array_elements(j, depth, std::integral_constant()); + write_cbor_container_head(0x80, j.m_data.m_value.array->size()); + if (!j.m_data.m_value.array->empty()) + { + write_cbor_array_elements(j, depth, std::integral_constant()); + } } + JSON_HEDLEY_NEVER_INLINE void write_cbor_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/) { write_array_elements(j, [this](CharType * out, const BasicJsonType & el) { return encode_cbor_integer(out, el); + }, [this](CharType * out, const number_float_t n) + { + return encode_cbor_float(out, n); }, [this, depth](const BasicJsonType & el) { write_cbor(el, depth + 1); @@ -26829,22 +29028,28 @@ class binary_writer /*! @brief write the elements of the array @a j, runs of integers with - @a encode_integer and all other values with @a write_value + @a encode_integer, runs of floats with @a encode_float, and all + other values with @a write_value - Runs of integers are encoded straight into space reserved from the sink + Runs of numbers are encoded straight into space reserved from the sink for a whole run: the write pointer stays a local, where it would otherwise be reloaded from the sink for every value, as a store of a character may alias it. */ - template - void write_array_elements(const BasicJsonType& j, EncodeInteger encode_integer, WriteValue write_value) + template + void write_array_elements(const BasicJsonType& j, EncodeInteger encode_integer, EncodeFloat encode_float, WriteValue write_value) { constexpr std::size_t max_run = max_reserve_size / integer_encoding_space; auto it = j.m_data.m_value.array->begin(); const auto end = j.m_data.m_value.array->end(); while (it != end) { - if (it->is_number_integer()) + if (!it->is_number()) + { + write_value(*it); + ++it; + } + else if (!it->is_number_float()) { CharType* const start = oa.reserve(max_run * integer_encoding_space); CharType* out = start; @@ -26856,12 +29061,33 @@ class binary_writer } else { - write_value(*it); - ++it; + it = write_float_elements(it, end, encode_float); } } } + /*! + @brief write the run of floats that starts at @a it with @a encode_float + @return the first element after the run + + Out of line, so that the loop of integers in @ref write_array_elements is + not changed by it. + */ + template + JSON_HEDLEY_NEVER_INLINE + Iterator write_float_elements(Iterator it, const Iterator end, EncodeFloat encode_float) // NOLINT(performance-unnecessary-value-param): iterators are passed by value + { + constexpr std::size_t max_run = max_reserve_size / integer_encoding_space; + CharType* const start = oa.reserve(max_run * integer_encoding_space); + CharType* out = start; + for (std::size_t n = 0; n < max_run && it != end && it->is_number_float(); ++n, ++it) + { + out += encode_float(out, it->m_data.m_value.number_float); + } + oa.commit(static_cast(out - start)); + return it; + } + /*! @brief write a number_integer or number_unsigned value as a CBOR integer @@ -27037,6 +29263,64 @@ class binary_writer } } + /// write a float as MessagePack, into space reserved from the sink + JSON_HEDLEY_NEVER_INLINE + void write_msgpack_float(const number_float_t n) + { + oa.commit(encode_compact_float(oa.reserve(integer_encoding_space), n, to_char_type(0xCA), to_char_type(0xCB))); + } + + /*! + @brief write an object value (head and members) as MessagePack + + See @ref write_cbor_object. + */ + JSON_HEDLEY_NEVER_INLINE + void write_msgpack_object(const BasicJsonType& j, const std::size_t depth) + { + write_msgpack_object_prefix(j.m_data.m_value.object->size(), j); + + for (const auto& el : *j.m_data.m_value.object) + { + // as in write_cbor, el.first is written directly with the + // object as diagnostics context + write_msgpack_key(el.first, j); + + const BasicJsonType& value = el.second; + switch (value.type()) + { + case value_t::null: + oa.write_character(to_char_type(0xC0)); + break; + + case value_t::boolean: + oa.write_character(value.m_data.m_value.boolean ? to_char_type(0xC3) : to_char_type(0xC2)); + break; + + case value_t::number_integer: + case value_t::number_unsigned: + write_msgpack_integer(value, std::integral_constant()); + break; + + case value_t::number_float: + write_msgpack_float(value.m_data.m_value.number_float); + break; + + case value_t::string: + write_msgpack_string(*value.m_data.m_value.string, value); + break; + + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + write_msgpack(value, depth + 1); + break; + } + } + } + /*! @brief write an array value (head and elements) as MessagePack @@ -27046,14 +29330,21 @@ class binary_writer void write_msgpack_array(const BasicJsonType& j, const std::size_t depth) { write_msgpack_array_prefix(j.m_data.m_value.array->size(), j); - write_msgpack_array_elements(j, depth, std::integral_constant()); + if (!j.m_data.m_value.array->empty()) + { + write_msgpack_array_elements(j, depth, std::integral_constant()); + } } + JSON_HEDLEY_NEVER_INLINE void write_msgpack_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/) { write_array_elements(j, [this](CharType * out, const BasicJsonType & el) { return encode_msgpack_integer(out, el); + }, [this](CharType * out, const number_float_t n) + { + return encode_compact_float(out, n, to_char_type(0xCA), to_char_type(0xCB)); }, [this, depth](const BasicJsonType & el) { write_msgpack(el, depth + 1); @@ -27105,6 +29396,24 @@ class binary_writer write_msgpack(BasicJsonType(key)); } + /// the longest string that is written, with its head of one or two + /// characters, in a single reservation from the sink + static constexpr std::size_t max_text_in_reservation = max_reserve_size - 2; + + /// copy the characters of a string to space in the sink; a short one + /// without a call of memcpy() + static void copy_characters(CharType* dst, const CharType* src, const std::size_t length) noexcept + { + if (length <= 32) + { + copy_short(dst, src, length, std::integral_constant()); + } + else + { + std::memcpy(dst, src, length * sizeof(CharType)); + } + } + /*! @brief write a CBOR text string @@ -27120,6 +29429,37 @@ class binary_writer this function. */ void write_cbor_string(const string_t& value, const BasicJsonType& context) + { + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + // the string is written as it is: head and characters in one reservation + if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 0x17) + { + *p++ = to_char_type(static_cast(0x60 + value.size())); + } + else + { + *p++ = to_char_type(0x78); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + return; + } + } + + write_cbor_checked_string(value, context); + } + + /// @brief write a CBOR text string checked or sanitized by the error handler; + /// out of line, as the string_t for the sanitized copy would make + /// every call of @ref write_cbor_string set up a frame for it + JSON_HEDLEY_NEVER_INLINE + void write_cbor_checked_string(const string_t& value, const BasicJsonType& context) { string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -27148,6 +29488,38 @@ class binary_writer @ref sanitize_utf8_for_write must not escape this function. */ void write_msgpack_string(const string_t& value, const BasicJsonType& context) + { + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + // the string is written as it is: head and characters in one reservation + if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 31) + { + // fixstr + *p++ = to_char_type(static_cast(0xA0 | value.size())); + } + else + { + // str 8 + *p++ = to_char_type(0xD9); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + return; + } + } + + write_msgpack_checked_string(value, context); + } + + /// @brief write a MessagePack str checked or sanitized by the error handler; + /// out of line, like @ref write_cbor_checked_string + JSON_HEDLEY_NEVER_INLINE + void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context) { string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -27217,6 +29589,375 @@ class binary_writer write_ubjson_integer_payload(prefix, n, use_bjdata); } + /*! + @brief markers and size classes of UBJSON/BJData integers, by the bit length + of the magnitude + + A non-negative integer has the magnitude n, a negative one ~n = -1 - n; the + bit length of the magnitude is 1 to 64. The first 130 characters of the + table are the markers: 65 for a non-negative integer, then 65 for a negative + one (index + 65). The marker is the first of `i`, `U`, `I`, `u` (BJData), + `l`, `m` (BJData), `L`, `M` (BJData) whose range contains the integer; `H` + (a high-precision number) is for a non-negative integer of 64 bits in + UBJSON. The following 130 characters are the size classes at index + 130: + the digit 0, 1, 2, or 3 for 1, 2, 4, or 8 bytes. Looked up, like + @ref integer_size_class, instead of found by branches or comparisons. + + @sa ubjson_integer_prefix for the integers of types of more than 64 bits + */ + static const char* ubjson_integer_table(const bool use_bjdata) noexcept + { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): initialized from string literals + static constexpr char ubjson_table[] = + // the markers for n >= 0, then n < 0 + "i" "iiiiiii" "U" "IIIIIII" "llllllllllllllll" "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL" "H" + "i" "iiiiiii" "IIIIIIII" "llllllllllllllll" "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL" "L" + // the size classes + "000000000" "1111111" "2222222222222222" "333333333333333333333333333333333" + "00000000" "11111111" "2222222222222222" "333333333333333333333333333333333"; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays): initialized from string literals + static constexpr char bjdata_table[] = + "i" "iiiiiii" "U" "IIIIIII" "u" "lllllllllllllll" "m" "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL" "M" + "i" "iiiiiii" "IIIIIIII" "llllllllllllllll" "LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL" "L" + "000000000" "11111111" "2222222222222222" "33333333333333333333333333333333" + "00000000" "11111111" "2222222222222222" "333333333333333333333333333333333"; + static_assert(sizeof(ubjson_table) == 261 && sizeof(bjdata_table) == 261, "two tables of 130 markers and 130 size classes"); + return use_bjdata ? bjdata_table : ubjson_table; + } + + /// the index in ubjson_integer_table() of the integer with the 64 bits + /// @a raw and the sign mask @a sign_mask (all ones if it is negative) + static std::size_t ubjson_integer_index(const std::uint64_t raw, const std::uint64_t sign_mask) noexcept + { + const std::size_t sign_offset = sign_mask != 0 ? std::size_t{65} : + std::size_t{0}; + return static_cast(64 - count_leading_zeros((raw ^ sign_mask) | 1u)) + sign_offset; + } + + /*! + @brief encode a UBJSON/BJData integer without branching on its size + @param[in] out where the encoding goes; at most 9 characters are written + @param[in] raw the 64 bits of the integer (two's complement) + @param[in] sign_mask all ones if the integer is negative, else zero + @param[in] table ubjson_integer_table(use_bjdata) + @return the end of the encoding, or nullptr if the integer is a high-precision + number (a non-negative one of 64 bits in UBJSON), which is not written + */ + CharType* encode_ubjson_integer(CharType* out, const std::uint64_t raw, const std::uint64_t sign_mask, + const bool add_prefix, const bool use_bjdata, const char* table) const noexcept + { + const std::size_t index = ubjson_integer_index(raw, sign_mask); + const char marker = table[index]; + if (JSON_HEDLEY_UNLIKELY(marker == 'H')) + { + return nullptr; + } + const unsigned size_class = static_cast(table[index + 130]) & 3u; + + // the marker (or its place, if not written) and one 8-byte word, whose + // bytes after the integer are scratch that the caller drops + out[0] = to_char_type(static_cast(marker)); + out += add_prefix ? 1 : 0; + const unsigned unused_bits = 64u - (8u << size_class); + // UBJSON in network order, BJData in little-endian + std::uint64_t tail = 0; + if (use_bjdata) + { + tail = is_little_endian ? raw : byteswap(raw); + } + else + { + tail = is_little_endian ? byteswap(raw) >> unused_bits : raw << unused_bits; + } + std::memcpy(out, &tail, sizeof(tail)); + return out + (std::size_t{1} << size_class); + } + + /// @brief whether the integers of BasicJsonType are written by + /// @ref encode_ubjson_integer + static constexpr bool ubjson_integers_fit_64_bits() noexcept + { + return integers_fit_64_bits(); + } + + /// space for the largest encoding of encode_ubjson_integer(): the marker and 8 bytes + static constexpr std::size_t ubjson_integer_encoding_space = 9; + + /*! + @brief write the bytes of a binary value as an array of UBJSON uint8 values + + Every byte is written with its marker, in blocks that fit the space reserved + from the sink. Kept out of line, so that write_ubjson does not carry the + registers of its loop. + */ + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_binary_bytes(const binary_t& binary, const bool bjdata_draft3) + { + const CharType marker = to_char_type(bjdata_draft3 ? 'B' : 'U'); + const auto* const data = binary.data(); + const std::size_t size = binary.size(); + for (std::size_t i = 0; i < size;) + { + const std::size_t block = (std::min)(size - i, max_reserve_size / 2); + CharType* const out = oa.reserve(2 * block); + for (std::size_t k = 0; k < block; ++k) + { + out[2 * k] = marker; + // the cast is needed for binary types whose value type + // is not an integer (e.g., std::byte) + out[(2 * k) + 1] = to_char_type(static_cast(data[i + k])); + } + i += block; + oa.commit(2 * block); + } + } + + /*! + @brief write the members of the object @a j of a UBJSON/BJData container + + A member whose value is null, a boolean, an integer, a float, or a string is + written without a call of @ref write_ubjson. Kept out of line, like + @ref write_cbor_object. + */ + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_members(const BasicJsonType& j, const bool use_count, const bool use_type, + const bool prefix_required, const bool use_bjdata, const bjdata_version_t bjdata_version, + const std::size_t depth) + { + for (const auto& el : *j.m_data.m_value.object) + { + // a converted key must outlive the reference returned by sanitize_utf8_for_write + const object_key_string_t key_string = el.first; + write_ubjson_text(key_string, j, false, use_bjdata); + + const BasicJsonType& value = el.second; + switch (value.type()) + { + case value_t::null: + if (prefix_required) + { + oa.write_character(to_char_type('Z')); + } + break; + + case value_t::boolean: + if (prefix_required) + { + oa.write_character(value.m_data.m_value.boolean ? to_char_type('T') : to_char_type('F')); + } + break; + + case value_t::number_integer: + write_number_with_ubjson_prefix(value.m_data.m_value.number_integer, prefix_required, use_bjdata); + break; + + case value_t::number_unsigned: + write_number_with_ubjson_prefix(value.m_data.m_value.number_unsigned, prefix_required, use_bjdata); + break; + + case value_t::number_float: + write_number_with_ubjson_prefix(value.m_data.m_value.number_float, prefix_required, use_bjdata); + break; + + case value_t::string: + write_ubjson_text(*value.m_data.m_value.string, value, prefix_required, use_bjdata); + break; + + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + write_ubjson(value, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + break; + } + } + } + + /// write the count marker `#` and the number of elements of a container as + /// an integer; a count of up to 127, which is `i` and a byte, in one reservation + void write_ubjson_count(const std::size_t count, const bool use_bjdata) + { + if (JSON_HEDLEY_LIKELY(count <= 127)) + { + CharType* const out = oa.reserve(3); + out[0] = to_char_type('#'); + out[1] = to_char_type('i'); + out[2] = to_char_type(static_cast(count)); + oa.commit(3); + return; + } + oa.write_character(to_char_type('#')); + write_number_with_ubjson_prefix(count, true, use_bjdata); + } + + /*! + @brief write a UBJSON/BJData string or object key: the optional string + marker `S`, the length as an integer with its marker, and the characters + + With the default error handler (keep) and a string of up to 253 characters, + which has a length of the marker `i` or `U` and a single byte, everything is + written into space reserved from the sink in one go. + + @param[in] value the string; checked or sanitized according to + @ref error_handler, with @a context as diagnostics context + @param[in] add_marker whether `S` is written first (not for an object key) + */ + void write_ubjson_text(const string_t& value, const BasicJsonType& context, const bool add_marker, const bool use_bjdata) + { + const std::size_t size = value.size(); + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep) && JSON_HEDLEY_LIKELY(size <= max_text_in_reservation - 1)) + { + CharType* const out = oa.reserve(size + 3); + CharType* p = out; + if (add_marker) + { + *p++ = to_char_type('S'); + } + *p++ = to_char_type(size <= 127 ? 'i' : 'U'); + *p++ = to_char_type(static_cast(size)); + copy_characters(p, reinterpret_cast(value.data()), size); + oa.commit(static_cast(p - out) + size); + return; + } + + write_ubjson_checked_text(value, context, add_marker, use_bjdata); + } + + /// @brief write a UBJSON/BJData string checked or sanitized by the error + /// handler; out of line, like @ref write_cbor_checked_string + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_checked_text(const string_t& value, const BasicJsonType& context, const bool add_marker, const bool use_bjdata) + { + string_t storage; + const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); + if (add_marker) + { + oa.write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(sanitized.size(), true, use_bjdata); + oa.write_characters( + reinterpret_cast(sanitized.data()), + sanitized.size()); + } + + /*! + @brief encode a UBJSON/BJData float, with its marker if @a add_prefix + @param[in] out where the encoding goes; at most 9 characters are written + @return the end of the encoding + */ + CharType* encode_ubjson_float(CharType* out, const number_float_t n, const bool add_prefix, const bool use_bjdata) const noexcept + { + if (add_prefix) + { + *out++ = get_ubjson_float_prefix(); + } + return store_number(out, n, use_bjdata); + } + + /// whether @a array starts with a number that @ref write_ubjson_array_elements + /// can encode (all but an integer that UBJSON writes as a high-precision + /// number); an array that does not is written by write_ubjson one element at a time + template + static bool starts_with_ubjson_number(const ArrayType& array, const bool use_bjdata, std::true_type /*fits_64_bits*/) noexcept + { + if (array.empty() || !array.front().is_number()) + { + return false; + } + const BasicJsonType& first = array.front(); + return use_bjdata || first.m_data.m_type != value_t::number_unsigned + || (static_cast(first.m_data.m_value.number_unsigned) >> 63u) == 0; + } + + template + static bool starts_with_ubjson_number(const ArrayType& /*array*/, const bool /*use_bjdata*/, std::false_type /*fits_64_bits*/) noexcept + { + return false; + } + + /*! + @brief write the elements of an array of a UBJSON/BJData container that + starts with a number, with runs of numbers encoded without a call of + @ref write_ubjson for each + + Same output as calling write_ubjson for each element. The numbers of a + run are encoded straight into space reserved from the sink, as in + @ref write_array_elements; all other values, and an integer that UBJSON + writes as a high-precision number, are written with write_ubjson. Kept out + of line, like @ref write_cbor_array. + */ + template + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_array_elements(const ArrayType& array, const bool use_count, const bool use_type, + const bool prefix_required, const bool use_bjdata, const bjdata_version_t bjdata_version, + const std::size_t depth) + { + constexpr std::size_t max_run = max_reserve_size / ubjson_integer_encoding_space; + const char* const table = ubjson_integer_table(use_bjdata); + auto it = array.begin(); + const auto end = array.end(); + while (it != end) + { + if (!it->is_number()) + { + write_ubjson(*it, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + ++it; + continue; + } + + CharType* const start = oa.reserve(max_run * ubjson_integer_encoding_space); + CharType* out = start; + bool unencodable = false; + for (std::size_t n = 0; n < max_run && it != end && it->is_number(); ++n) + { + CharType* next = nullptr; + if (it->is_number_float()) + { + next = encode_ubjson_float(out, it->m_data.m_value.number_float, prefix_required, use_bjdata); + } + else + { + std::uint64_t sign_mask = 0; + const std::uint64_t raw = integer_bits(*it, sign_mask); + next = encode_ubjson_integer(out, raw, sign_mask, prefix_required, use_bjdata, table); + } + if (JSON_HEDLEY_UNLIKELY(next == nullptr)) + { + unencodable = true; + break; + } + out = next; + ++it; + } + oa.commit(static_cast(out - start)); + + if (unencodable) + { + write_ubjson(*it, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); + ++it; + } + } + } + + /*! + @brief store @a n at @a out in the byte order of the output + @param[in] output_little_endian whether the output is little-endian + @return the end of the number + */ + template + CharType* store_number(CharType* out, const NumberType n, const bool output_little_endian) const noexcept + { + std::array vec{}; + std::memcpy(vec.data(), &n, sizeof(NumberType)); + if (is_little_endian != output_little_endian) + { + reverse_bytes(vec); + } + std::memcpy(out, vec.data(), sizeof(NumberType)); + return out + sizeof(NumberType); + } + /*! @brief determine the UBJSON/BJData type marker of an integer @@ -27307,19 +30048,58 @@ class binary_writer break; default: { - // high-precision number: the decimal digits as a string JSON_ASSERT(prefix == 'H'); - const auto number = BasicJsonType(n).dump(); - write_number_with_ubjson_prefix(number.size(), true, use_bjdata); - for (std::size_t i = 0; i < number.size(); ++i) - { - oa.write_character(to_char_type(static_cast(number[i]))); - } + write_ubjson_high_precision(n, use_bjdata); break; } } } + /// @brief write an integer as a high-precision number: its decimal digits as + /// a string; out of line, as the string would otherwise burden + /// @ref write_ubjson_integer_payload for every integer + template + JSON_HEDLEY_NEVER_INLINE + void write_ubjson_high_precision(const NumberType n, const bool use_bjdata) + { + write_ubjson_high_precision(n, use_bjdata, std::integral_constant < bool, std::is_unsigned::value && sizeof(NumberType) <= sizeof(std::uint64_t) > ()); + } + + /// an unsigned integer of at most 64 bits: its decimal digits are written directly + template + void write_ubjson_high_precision(const NumberType n, const bool use_bjdata, std::true_type /*is_unsigned_64_bits*/) + { + std::array digits; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::size_t first = digits.size(); + auto value = static_cast(n); + do + { + digits[--first] = static_cast('0' + (value % 10)); + value /= 10; + } + while (value != 0); + const std::size_t length = digits.size() - first; + write_number_with_ubjson_prefix(length, true, use_bjdata); + CharType* const out = oa.reserve(digits.size()); + for (std::size_t i = 0; i < length; ++i) + { + out[i] = to_char_type(static_cast(digits[first + i])); + } + oa.commit(length); + } + + /// any other integer: its decimal digits from basic_json::dump() + template + void write_ubjson_high_precision(const NumberType n, const bool use_bjdata, std::false_type /*is_unsigned_64_bits*/) + { + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true, use_bjdata); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa.write_character(to_char_type(static_cast(number[i]))); + } + } + /*! @brief determine the type prefix of container values */ @@ -27397,8 +30177,8 @@ class binary_writer static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) { return el.is_number_unsigned() - ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(el.template get()) - : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(el.template get()); + ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(static_cast(el.m_data.m_value.number_unsigned)) + : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(static_cast(el.m_data.m_value.number_integer)); } /*! @@ -27487,7 +30267,7 @@ class binary_writer */ bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run) { - const auto dval = el.template get(); + const auto dval = static_cast(el.m_data.m_value.number_float); if (dry_run) { #ifdef __GNUC__ @@ -27511,6 +30291,90 @@ class binary_writer return true; } + /*! + @brief write the elements of an ND-array as the number kind @a dtype names + (see @ref write_bjdata_ndarray), after they have been found to be + in range by a dry run of @ref write_bjdata_ndarray_elements + + The elements are encoded in blocks into space reserved from the sink. + */ + void write_bjdata_ndarray_values(const BasicJsonType& array_data, const CharType dtype) + { + switch (dtype) + { + case 'U': + case 'C': + case 'B': + write_bjdata_ndarray_block(array_data); + break; + case 'i': + write_bjdata_ndarray_block(array_data); + break; + case 'u': + write_bjdata_ndarray_block(array_data); + break; + case 'I': + write_bjdata_ndarray_block(array_data); + break; + case 'm': + write_bjdata_ndarray_block(array_data); + break; + case 'l': + write_bjdata_ndarray_block(array_data); + break; + case 'M': + write_bjdata_ndarray_block(array_data); + break; + case 'L': + write_bjdata_ndarray_block(array_data); + break; + case 'd': + write_bjdata_ndarray_block(array_data); + break; + case 'D': + default: + // 'D' (double) already spans the full range of number_float_t + write_bjdata_ndarray_block(array_data); + break; + } + } + + /// the value of an ND-array element that is an integer, as the integer type that @a T is converted from + template + static T bjdata_ndarray_integer(const BasicJsonType& el, std::true_type /*is_integral*/) noexcept + { + using storage_type = typename std::conditional::value, std::uint64_t, std::int64_t>::type; + return static_cast(el.is_number_unsigned() + ? static_cast(el.m_data.m_value.number_unsigned) + : static_cast(el.m_data.m_value.number_integer)); + } + + /// the value of an ND-array element that is a float + template + static T bjdata_ndarray_integer(const BasicJsonType& el, std::false_type /*is_integral*/) noexcept + { + return static_cast(static_cast(el.m_data.m_value.number_float)); + } + + /// write the elements of an ND-array as @a T, little-endian, in blocks + template + void write_bjdata_ndarray_block(const BasicJsonType& array_data) + { + constexpr std::size_t per_block = max_reserve_size / sizeof(T); + auto it = array_data.m_data.m_value.array->begin(); + const auto end = array_data.m_data.m_value.array->end(); + while (it != end) + { + CharType* const start = oa.reserve(per_block * sizeof(T)); + CharType* out = start; + for (std::size_t n = 0; n < per_block && it != end; ++n, ++it) + { + out = store_number(out, bjdata_ndarray_integer(*it, std::is_integral()), true); + } + oa.commit(static_cast(out - start)); + } + } + /*! @brief validate or write every element of a BJData ND-array's `_ArrayData_` @param[in] array_data the `_ArrayData_` array @@ -27520,6 +30384,12 @@ class binary_writer */ bool write_bjdata_ndarray_elements(const BasicJsonType& array_data, const CharType dtype, const bool dry_run) { + if (!dry_run) + { + write_bjdata_ndarray_values(array_data, dtype); + return true; + } + for (const auto& el : array_data) { bool ok = true; @@ -27805,9 +30675,16 @@ class binary_writer // 0x80..0x84: array with 0..4 elements; 0x85: array ended by 0xFE write_bon8_marker(static_cast(N <= 4 ? 0x80 + N : 0x85), string_open); - for (const auto& el : *j.m_data.m_value.array) + if (N != 0 && j.m_data.m_value.array->front().is_number()) { - write_bon8_value(el, string_open, depth + 1); + write_bon8_array_elements(*j.m_data.m_value.array, string_open, depth); + } + else + { + for (const auto& el : *j.m_data.m_value.array) + { + write_bon8_value(el, string_open, depth + 1); + } } if (N > 4) @@ -27823,11 +30700,7 @@ class binary_writer // 0x86..0x8A: object with 0..4 members; 0x8B: object ended by 0xFE write_bon8_marker(static_cast(N <= 4 ? 0x86 + N : 0x8B), string_open); - for (const auto& el : *j.m_data.m_value.object) - { - write_bon8_string(el.first, string_open, j); - write_bon8_value(el.second, string_open, depth + 1); - } + write_bon8_members(j, string_open, depth); if (N > 4) { @@ -27838,22 +30711,7 @@ class binary_writer case value_t::binary: { - // BON8 has no binary type: write the bytes as an array of - // integers, like UBJSON and BJData do - const auto N = j.m_data.m_value.binary->size(); - write_bon8_marker(static_cast(N <= 4 ? 0x80 + N : 0x85), string_open); - - for (std::size_t i = 0; i < N; ++i) - { - // the cast is needed for binary types whose value type - // is not an integer (e.g., std::byte) - write_bon8_integer(static_cast(j.m_data.m_value.binary->data()[i])); - } - - if (N > 4) - { - write_bon8_marker(0xFE, string_open); - } + write_bon8_binary(*j.m_data.m_value.binary, string_open); break; } @@ -27940,6 +30798,281 @@ class binary_writer } } + /*! + @brief encode an integer as @ref write_bon8_integer writes it + @param[in] out where the encoding goes; at most 9 characters are written + @return the end of the encoding + */ + CharType* encode_bon8_integer(CharType* out, std::int64_t value) const noexcept + { + if (value < (std::numeric_limits::min)() || value > (std::numeric_limits::max)()) + { + *out++ = to_char_type(0x8D); + return store_number(out, value, false); + } + if (value < -33818506 || value > 67637031) + { + *out++ = to_char_type(0x8C); + return store_number(out, static_cast(value), false); + } + if (value <= -264075) + { + value = -(value + 264075); + return put_bon8_bytes(out, 0xF0 + ((value >> 22) & 0x07), 0xC0 + ((value >> 16) & 0x3F), value >> 8, value); + } + if (value <= -1931) + { + value = -(value + 1931); + return put_bon8_bytes(out, 0xE0 + ((value >> 14) & 0x0F), 0xC0 + ((value >> 8) & 0x3F), value); + } + if (value <= -11) + { + value = -(value + 11); + return put_bon8_bytes(out, 0xC2 + ((value >> 6) & 0x1F), 0xC0 + (value & 0x3F)); + } + if (value <= -1) + { + return put_bon8_bytes(out, 0xB8 - (value + 1)); + } + if (value <= 39) + { + return put_bon8_bytes(out, 0x90 + value); + } + if (value <= 3879) + { + value -= 40; + return put_bon8_bytes(out, 0xC2 + ((value >> 7) & 0x1F), value & 0x7F); + } + if (value <= 528167) + { + value -= 3880; + return put_bon8_bytes(out, 0xE0 + ((value >> 15) & 0x0F), (value >> 8) & 0x7F, value); + } + value -= 528168; + return put_bon8_bytes(out, 0xF0 + ((value >> 23) & 0x07), (value >> 16) & 0x7F, value >> 8, value); + } + + /// store the low byte of each argument at @a out; return the end + template + CharType* put_bon8_bytes(CharType* out, const Bytes... bytes) const noexcept + { + const std::array buffer{{to_char_type(static_cast(bytes & 0xFF))...}}; + std::memcpy(out, buffer.data(), buffer.size()); + return out + buffer.size(); + } + + /*! + @brief encode a float as @ref write_bon8_float writes it + @param[in] out where the encoding goes; at most 9 characters are written + @return the end of the encoding + */ + CharType* encode_bon8_float(CharType* out, const number_float_t n) const noexcept + { +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + if (n == static_cast(-1)) + { + *out++ = to_char_type(0xFB); + return out; + } + if (n == static_cast(0) && !std::signbit(n)) + { + *out++ = to_char_type(0xFC); + return out; + } + if (n == static_cast(1)) + { + *out++ = to_char_type(0xFD); + return out; + } + if (std::isnan(n)) + { + return put_bon8_bytes(out, 0x8E, 0x7F, 0x80, 0x00, 0x01); + } + return out + encode_compact_float(out, n, to_char_type(0x8E), to_char_type(0x8F)); +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + } + + /*! + @brief write the elements of an array that starts with a number, with runs + of numbers encoded without a call of @ref write_bon8_value for each + + Same output as calling write_bon8_value for each element. The numbers of a + run are encoded into space reserved from the sink, as in + @ref write_array_elements; an integer that BON8 cannot write (an unsigned + one above the int64 range, or one of a number type wider than 64 bits) throws + from write_bon8_value, like any other value that is not a number. Kept out + of line, like @ref write_cbor_array. + */ + template + JSON_HEDLEY_NEVER_INLINE + void write_bon8_array_elements(const ArrayType& array, bool& string_open, const std::size_t depth) + { + constexpr std::size_t max_run = max_reserve_size / 9; + auto it = array.begin(); + const auto end = array.end(); + while (it != end) + { + if (!it->is_number()) + { + write_bon8_value(*it, string_open, depth + 1); + ++it; + continue; + } + + CharType* const start = oa.reserve(max_run * 9); + CharType* out = start; + bool unencodable = false; + for (std::size_t n = 0; n < max_run && it != end && it->is_number(); ++n) + { + if (it->is_number_float()) + { + out = encode_bon8_float(out, it->m_data.m_value.number_float); + } + else if (JSON_HEDLEY_UNLIKELY((it->m_data.m_type == value_t::number_unsigned + && !integer_fits_max(it->m_data.m_value.number_unsigned)) + || (it->m_data.m_type == value_t::number_integer + && !integer_fits(it->m_data.m_value.number_integer)))) + { + unencodable = true; + break; + } + else + { + out = encode_bon8_integer(out, it->m_data.m_type == value_t::number_integer + ? static_cast(it->m_data.m_value.number_integer) + : static_cast(it->m_data.m_value.number_unsigned)); + } + ++it; + } + oa.commit(static_cast(out - start)); + if (out != start) + { + // a number follows a string without terminating it + string_open = false; + } + + if (unencodable) + { + write_bon8_value(*it, string_open, depth + 1); + ++it; + } + } + } + + /*! + @brief write the members of the object @a j as BON8 + + A member whose value is null, a boolean, a number, or a string is written + without a call of @ref write_bon8_value. Kept out of line, like + @ref write_cbor_object. + */ + JSON_HEDLEY_NEVER_INLINE + void write_bon8_members(const BasicJsonType& j, bool& string_open, const std::size_t depth) + { + for (const auto& el : *j.m_data.m_value.object) + { + write_bon8_string(el.first, string_open, j); + + const BasicJsonType& value = el.second; + switch (value.type()) + { + case value_t::null: + write_bon8_marker(0xFA, string_open); + break; + + case value_t::boolean: + write_bon8_marker(value.m_data.m_value.boolean ? 0xF9 : 0xF8, string_open); + break; + + case value_t::string: + write_bon8_string(*value.m_data.m_value.string, string_open, value); + break; + + case value_t::number_integer: + if (JSON_HEDLEY_LIKELY(integer_fits(value.m_data.m_value.number_integer))) + { + write_bon8_integer(static_cast(value.m_data.m_value.number_integer)); + string_open = false; + break; + } + // beyond 64 bits (number types wider than that): throws + write_bon8_value(value, string_open, depth + 1); + break; + + case value_t::number_float: + write_bon8_float(value.m_data.m_value.number_float); + string_open = false; + break; + + case value_t::number_unsigned: + if (JSON_HEDLEY_LIKELY(integer_fits_max(value.m_data.m_value.number_unsigned))) + { + write_bon8_integer(static_cast(value.m_data.m_value.number_unsigned)); + string_open = false; + break; + } + // too large for BON8: throws + write_bon8_value(value, string_open, depth + 1); + break; + + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + write_bon8_value(value, string_open, depth + 1); + break; + } + } + } + + /*! + @brief write a binary value as BON8 + + BON8 has no binary type: the bytes are written as an array of integers, + like UBJSON and BJData do. Kept out of line, so that write_bon8_value does + not carry the registers of its loop. + */ + JSON_HEDLEY_NEVER_INLINE + void write_bon8_binary(const binary_t& binary, bool& string_open) + { + const auto N = binary.size(); + write_bon8_marker(static_cast(N <= 4 ? 0x80 + N : 0x85), string_open); + + // a byte is an integer of 0 to 255: one byte up to 39, two bytes above + const auto* const data = binary.data(); + for (std::size_t i = 0; i < N;) + { + const std::size_t block = (std::min)(N - i, max_reserve_size / 2); + CharType* const start = oa.reserve(2 * block); + CharType* out = start; + for (std::size_t k = 0; k < block; ++k, ++i) + { + // the cast is needed for binary types whose value type + // is not an integer (e.g., std::byte) + const auto value = static_cast(static_cast(data[i])); + // both bytes are stored, the second only counts for a value above 39: + // 0xC2 + ((value - 40) >> 7), (value - 40) & 0x7F + const unsigned wide = value - 40u; + const bool is_wide = value > 39u; + out[0] = to_char_type(static_cast(is_wide ? 0xC2u + (wide >> 7u) : 0x90u + value)); + out[1] = to_char_type(static_cast(wide & 0x7Fu)); + out += is_wide ? 2 : 1; + } + oa.commit(static_cast(out - start)); + } + + if (N > 4) + { + write_bon8_marker(0xFE, string_open); + } + } + /*! @brief write a single byte that is not part of a string @@ -27997,15 +31130,25 @@ class binary_writer */ static void check_utf8(const string_t& s, const BasicJsonType& context) { - static_cast(context); // only used when exceptions are enabled const auto* data = reinterpret_cast(s.data()); const std::size_t valid = valid_utf8_prefix(data, s.size()); if (JSON_HEDLEY_UNLIKELY(valid != s.size())) { - JSON_THROW(type_error::create(exception_id::invalid_utf8, concat("invalid UTF-8 byte at index ", std::to_string(valid), ": 0x", detail::hex_byte(data[valid])), &context)); + throw_invalid_utf8(data, valid, context); } } + /// @brief throw type_error.316 for the invalid byte at @a index of @a data; + /// out of line, so that the message does not burden check_utf8() + JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE + static void throw_invalid_utf8(const unsigned char* data, const std::size_t index, const BasicJsonType& context) + { + static_cast(data); // only used when exceptions are enabled + static_cast(index); + static_cast(context); + JSON_THROW(type_error::create(exception_id::invalid_utf8, concat("invalid UTF-8 byte at index ", std::to_string(index), ": 0x", detail::hex_byte(data[index])), &context)); + } + /*! @brief return @a s as it should be written, honoring @ref error_handler @@ -28254,6 +31397,67 @@ class binary_writer oa.write_characters(vec.data(), sizeof(NumberType)); } + /// whether @a n is written as a float32 by @ref write_compact_float + static bool fits_float32(const number_float_t n) noexcept + { + static_assert(std::is_same::value || std::is_same::value, + "number_float_t must be float or double for the CBOR/MessagePack/BON8 writer"); +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_PUSH + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wduplicated-branches") +#endif + return !std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && + static_cast(n) <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(n)) == static_cast(n))); +#ifdef __GNUC__ + JSON_HEDLEY_DIAGNOSTIC_POP +#endif + } + + /*! + @brief encode @a n to @a out as @ref write_compact_float writes it + @return the number of characters, 5 or 9 + */ + std::size_t encode_compact_float(CharType* out, const number_float_t n, const CharType float32_marker, const CharType float64_marker) const noexcept + { + if (fits_float32(n)) + { + *out = float32_marker; + store_number(out + 1, static_cast(n), false); + return 1 + sizeof(float); + } + *out = float64_marker; + store_number(out + 1, n, false); + return 1 + sizeof(number_float_t); + } + + /*! + @brief encode a float as @ref write_cbor writes it + @return the number of characters, 3, 5 or 9 + */ + std::size_t encode_cbor_float(CharType* out, const number_float_t n) const noexcept + { + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(n))) + { + // NaN is 0xf97e00, Infinity is 0xf97c00, -Infinity is 0xf9fc00 + out[0] = to_char_type(0xF9); + if (std::isnan(n)) + { + out[1] = to_char_type(0x7E); + } + else + { + out[1] = n > 0 ? to_char_type(0x7C) : to_char_type(0xFC); + } + out[2] = to_char_type(0x00); + return 3; + } + return encode_compact_float(out, n, to_char_type(0xFA), to_char_type(0xFB)); + } + /// @brief write @a n using @a float32_marker if it round-trips through /// float, otherwise using @a float64_marker /// @@ -28263,24 +31467,7 @@ class binary_writer /// no longer needs to be passed in. void write_compact_float(const number_float_t n, const CharType float32_marker, const CharType float64_marker) { - static_assert(std::is_same::value || std::is_same::value, - "number_float_t must be float or double for the CBOR/MessagePack/BON8 writer"); -#ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_PUSH - JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") -#endif - // When number_float_t is float, static_cast(n) is the identity and - // both branches below are intentionally identical (the "compact" float - // representation is the value itself). Only GCC diagnoses this, and only - // when the sink calls are inlined; clang has no such warning. - // (-Wduplicated-branches only exists from GCC 7 on; naming it on an older - // GCC would itself warn under -Wpragmas) -#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) - JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wduplicated-branches") -#endif - if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && - static_cast(n) <= static_cast((std::numeric_limits::max)()) && - static_cast(static_cast(n)) == static_cast(n)))) + if (fits_float32(n)) { oa.write_character(float32_marker); write_number(static_cast(n)); @@ -28290,9 +31477,6 @@ class binary_writer oa.write_character(float64_marker); write_number(n); } -#ifdef __GNUC__ - JSON_HEDLEY_DIAGNOSTIC_POP -#endif } public: @@ -30215,7 +33399,7 @@ class serializer None of @a pretty_print_, @a ensure_ascii_ and @a indent_step_ change over the life of the serializer, so they are captured once here instead of - being threaded through every call to @ref dump, @ref dump_internal and + being threaded through every call to @ref dump, @ref dump_container and @ref dump_iteratively. */ serializer(output_adapter_protocol& s, const char ichar, @@ -30224,11 +33408,14 @@ class serializer const std::size_t indent_step_ = 0, error_handler_t error_handler_ = error_handler_t::strict) : o(&s) + , number_buffer() // NOLINT(readability-redundant-member-init) + , string_buffer() // NOLINT(readability-redundant-member-init) , indent_char(ichar) , pretty_print(pretty_print_) , ensure_ascii(ensure_ascii_) , indent_step(indent_step_) , error_handler(error_handler_) + , write_buffer() // NOLINT(readability-redundant-member-init) {} // deleted because of pointer members @@ -30259,18 +33446,18 @@ class serializer void dump(const BasicJsonType& val, const std::size_t current_indent = 0) { - dump_internal(val, current_indent); + dump_element(val, current_indent, 0); flush(); } JSON_PRIVATE_UNLESS_TESTED: /*! - @brief worker for @ref dump + @brief serialize the object or array @a val - Identical in behavior to the historical @ref dump, but writes into the - serializer's internal @ref write_buffer instead of issuing a virtual call - per token. The public @ref dump wraps this and flushes the buffer once the - top-level value has been serialized. + Writes into the serializer's internal @ref write_buffer instead of issuing + a virtual call per token; the public @ref dump flushes the buffer once the + top-level value has been serialized. Scalars are written by @ref dump_scalar, + which @ref dump_element calls without entering this function. Serializing a container descends into its elements, so a value nested deeply enough used to exhaust the call stack and terminate the process with no @@ -30281,167 +33468,149 @@ class serializer @sa https://github.com/nlohmann/json/issues/5387 */ - void dump_internal(const BasicJsonType& val, - const std::size_t current_indent = 0, - const std::size_t depth = 0) + void dump_container(const BasicJsonType& val, + const std::size_t current_indent, + const std::size_t depth) { - switch (val.m_data.m_type) + JSON_ASSERT(val.m_data.m_type == value_t::object || val.m_data.m_type == value_t::array); + + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) { - case value_t::object: + dump_iteratively(val, current_indent); + return; + } + + if (val.m_data.m_type == value_t::object) + { + if (val.m_data.m_value.object->empty()) { - if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) - { - dump_iteratively(val, current_indent); - return; - } - - if (val.m_data.m_value.object->empty()) - { - put_literal("{}"); - return; - } - - if (pretty_print) - { - put_literal("{\n"); - - // variable to hold indentation for recursive calls - const auto new_indent = next_indent(current_indent, indent_step); - - // first n-1 elements - auto i = val.m_data.m_value.object->cbegin(); - for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) - { - put_indent(new_indent); - put_char('"'); - dump_escaped(i->first); - put_literal("\": "); - dump_internal(i->second, new_indent, depth + 1); - put_literal(",\n"); - } - - // last element - JSON_ASSERT(i != val.m_data.m_value.object->cend()); - JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - put_indent(new_indent); - put_char('"'); - dump_escaped(i->first); - put_literal("\": "); - dump_internal(i->second, new_indent, depth + 1); - - put_char('\n'); - put_indent(current_indent); - put_char('}'); - } - else - { - put_char('{'); - - // first n-1 elements - auto i = val.m_data.m_value.object->cbegin(); - for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) - { - put_char('"'); - dump_escaped(i->first); - put_literal("\":"); - dump_internal(i->second, current_indent, depth + 1); - put_char(','); - } - - // last element - JSON_ASSERT(i != val.m_data.m_value.object->cend()); - JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - put_char('"'); - dump_escaped(i->first); - put_literal("\":"); - dump_internal(i->second, current_indent, depth + 1); - - put_char('}'); - } - + put_literal("{}"); return; } - case value_t::array: + if (pretty_print) { - if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) + // variable to hold indentation for recursive calls + const auto new_indent = next_indent(current_indent, indent_step); + + put_literal_indent("{\n", new_indent); + + // first n-1 elements + auto i = val.m_data.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - dump_iteratively(val, current_indent); - return; + put_object_key(i->first, "\": "); + dump_element(i->second, new_indent, depth + 1); + put_literal_indent(",\n", new_indent); } - if (val.m_data.m_value.array->empty()) + // last element + JSON_ASSERT(i != val.m_data.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); + put_object_key(i->first, "\": "); + dump_element(i->second, new_indent, depth + 1); + + put_literal_indent("\n", current_indent); + put_char('}'); + } + else + { + put_char('{'); + + // first n-1 elements + auto i = val.m_data.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - put_literal("[]"); - return; + put_object_key(i->first, "\":"); + dump_element(i->second, 0, depth + 1); + put_char(','); } - if (pretty_print) - { - put_literal("[\n"); + // last element + JSON_ASSERT(i != val.m_data.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); + put_object_key(i->first, "\":"); + dump_element(i->second, 0, depth + 1); - // variable to hold indentation for recursive calls - const auto new_indent = next_indent(current_indent, indent_step); + put_char('}'); + } + return; + } - // first n-1 elements - for (auto i = val.m_data.m_value.array->cbegin(); - i != val.m_data.m_value.array->cend() - 1; ++i) - { - put_indent(new_indent); - dump_internal(*i, new_indent, depth + 1); - put_literal(",\n"); - } + if (val.m_data.m_value.array->empty()) + { + put_literal("[]"); + return; + } - // last element - JSON_ASSERT(!val.m_data.m_value.array->empty()); - put_indent(new_indent); - dump_internal(val.m_data.m_value.array->back(), new_indent, depth + 1); + if (pretty_print) + { + // variable to hold indentation for recursive calls + const auto new_indent = next_indent(current_indent, indent_step); - put_char('\n'); - put_indent(current_indent); - put_char(']'); - } - else - { - put_char('['); + put_literal_indent("[\n", new_indent); - // first n-1 elements - for (auto i = val.m_data.m_value.array->cbegin(); - i != val.m_data.m_value.array->cend() - 1; ++i) - { - dump_internal(*i, current_indent, depth + 1); - put_char(','); - } - - // last element - JSON_ASSERT(!val.m_data.m_value.array->empty()); - dump_internal(val.m_data.m_value.array->back(), current_indent, depth + 1); - - put_char(']'); - } - - return; + // first n-1 elements + for (auto i = val.m_data.m_value.array->cbegin(); + i != val.m_data.m_value.array->cend() - 1; ++i) + { + dump_element(*i, new_indent, depth + 1); + put_literal_indent(",\n", new_indent); } - case value_t::string: - case value_t::binary: - case value_t::boolean: - case value_t::number_integer: - case value_t::number_unsigned: - case value_t::number_float: - case value_t::discarded: - case value_t::null: - default: - dump_scalar(val, current_indent); - return; + // last element + JSON_ASSERT(!val.m_data.m_value.array->empty()); + dump_element(val.m_data.m_value.array->back(), new_indent, depth + 1); + + put_literal_indent("\n", current_indent); + put_char(']'); + } + else + { + put_char('['); + + // first n-1 elements + for (auto i = val.m_data.m_value.array->cbegin(); + i != val.m_data.m_value.array->cend() - 1; ++i) + { + dump_element(*i, 0, depth + 1); + put_char(','); + } + + // last element + JSON_ASSERT(!val.m_data.m_value.array->empty()); + dump_element(val.m_data.m_value.array->back(), 0, depth + 1); + + put_char(']'); } } private: + /*! + @brief serialize an element of a container + + Most elements are scalars, which need none of the state dump_container() keeps + for a container; writing them from here spares each the entry and exit of + that function. + */ + JSON_HEDLEY_ALWAYS_INLINE void dump_element(const BasicJsonType& val, + const std::size_t current_indent, + const std::size_t depth) + { + if (val.m_data.m_type == value_t::object || val.m_data.m_type == value_t::array) + { + dump_container(val, current_indent, depth); + } + else + { + dump_scalar(val, current_indent); + } + } + /*! @brief write out @a val and everything below it without the call stack - Emits the same bytes as @ref dump_internal, keeping the containers it has + Emits the same bytes as @ref dump_container, keeping the containers it has entered on an explicit stack instead of descending into them. Only reached for values nested deeper than @ref recursion_depth_limit, which is why it is not written for speed: walking every value this way measured up to 20% slower on @@ -30469,8 +33638,7 @@ class serializer { if (pretty_print) { - put_char('\n'); - put_indent(frame.current_indent); + put_literal_indent("\n", frame.current_indent); } put_char('}'); @@ -30497,16 +33665,13 @@ class serializer put_indent(frame.child_indent); } - put_char('"'); - dump_escaped(frame.object_it->first); - if (pretty_print) { - put_literal("\": "); + put_object_key(frame.object_it->first, "\": "); } else { - put_literal("\":"); + put_object_key(frame.object_it->first, "\":"); } const BasicJsonType& element = frame.object_it->second; @@ -30525,8 +33690,7 @@ class serializer { if (pretty_print) { - put_char('\n'); - put_indent(frame.current_indent); + put_literal_indent("\n", frame.current_indent); } put_char(']'); @@ -30667,7 +33831,7 @@ class serializer /*! @brief serialize the value @a val, which is neither an object nor an array - Shared by @ref dump_internal and @ref dump_value, so that a value is written + Shared by @ref dump_element and @ref dump_value, so that a value is written the same way however deeply it is nested. A binary value is written out here in full: it looks like an object, but has no elements to descend into. @@ -30681,78 +33845,13 @@ class serializer { case value_t::string: { - put_char('"'); - dump_escaped(*val.m_data.m_value.string); - put_char('"'); + dump_string_value(*val.m_data.m_value.string); return; } case value_t::binary: { - if (pretty_print) - { - put_literal("{\n"); - - // variable to hold indentation for the bytes - const auto new_indent = next_indent(current_indent, indent_step); - - put_indent(new_indent); - - put_literal("\"bytes\": ["); - - if (!val.m_data.m_value.binary->empty()) - { - for (auto i = val.m_data.m_value.binary->cbegin(); - i != val.m_data.m_value.binary->cend() - 1; ++i) - { - dump_byte(*i); - put_literal(", "); - } - dump_byte(val.m_data.m_value.binary->back()); - } - - put_literal("],\n"); - put_indent(new_indent); - - put_literal("\"subtype\": "); - if (val.m_data.m_value.binary->has_subtype()) - { - dump_integer(val.m_data.m_value.binary->subtype()); - } - else - { - put_literal("null"); - } - put_char('\n'); - put_indent(current_indent); - put_char('}'); - } - else - { - put_literal("{\"bytes\":["); - - if (!val.m_data.m_value.binary->empty()) - { - for (auto i = val.m_data.m_value.binary->cbegin(); - i != val.m_data.m_value.binary->cend() - 1; ++i) - { - dump_byte(*i); - put_char(','); - } - dump_byte(val.m_data.m_value.binary->back()); - } - - put_literal("],\"subtype\":"); - if (val.m_data.m_value.binary->has_subtype()) - { - dump_integer(val.m_data.m_value.binary->subtype()); - put_char('}'); - } - else - { - put_literal("null}"); - } - } + dump_binary_value(current_indent, *val.m_data.m_value.binary); return; } @@ -30771,19 +33870,19 @@ class serializer case value_t::number_integer: { - dump_integer(val.m_data.m_value.number_integer); + dump_signed_value(val.m_data.m_value.number_integer); return; } case value_t::number_unsigned: { - dump_integer(val.m_data.m_value.number_unsigned); + dump_unsigned_value(val.m_data.m_value.number_unsigned); return; } case value_t::number_float: { - dump_float(val.m_data.m_value.number_float); + dump_float_value(val.m_data.m_value.number_float); return; } @@ -30806,6 +33905,100 @@ class serializer } } + /// @brief write the string @a s in quotes; kept out of line, so that the + /// other scalars do not pay for its register needs (and the other way round) + JSON_HEDLEY_NEVER_INLINE void dump_string_value(const string_t& s) + { +#if JSON_LITTLE_ENDIAN_TARGET + if (s.size() <= short_string_max && put_short_quoted(s, "\"")) + { + return; + } +#endif + put_char('"'); + dump_escaped(s); + put_char('"'); + } + + // The numbers, and the rest of what dump_scalar() writes, are left to + // functions of their own as well: a single function holding all of them + // saves and restores every register any of them needs, on every call. Now + // dump_scalar() only dispatches, and each writer pays for what it uses. + JSON_HEDLEY_NEVER_INLINE void dump_signed_value(const number_integer_t x) + { + dump_integer(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_unsigned_value(const number_unsigned_t x) + { + dump_integer(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_float_value(const number_float_t x) + { + dump_float(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_binary_value(const std::size_t current_indent, const typename BasicJsonType::binary_t& binary) + { + if (pretty_print) + { + // variable to hold indentation for the bytes + const auto new_indent = next_indent(current_indent, indent_step); + + put_literal_indent("{\n", new_indent); + put_literal("\"bytes\": ["); + + if (!binary.empty()) + { + for (auto i = binary.cbegin(); i != binary.cend() - 1; ++i) + { + dump_byte(*i); + put_literal(", "); + } + dump_byte(binary.back()); + } + + put_literal_indent("],\n", new_indent); + + put_literal("\"subtype\": "); + if (binary.has_subtype()) + { + dump_integer(binary.subtype()); + } + else + { + put_literal("null"); + } + put_literal_indent("\n", current_indent); + put_char('}'); + } + else + { + put_literal("{\"bytes\":["); + + if (!binary.empty()) + { + for (auto i = binary.cbegin(); i != binary.cend() - 1; ++i) + { + dump_byte(*i); + put_char(','); + } + dump_byte(binary.back()); + } + + put_literal("],\"subtype\":"); + if (binary.has_subtype()) + { + dump_integer(binary.subtype()); + put_char('}'); + } + else + { + put_literal("null}"); + } + } + } /*! @brief the indentation level to use for the children of the current value @@ -30822,6 +34015,220 @@ class serializer return new_indent; } + private: + /*! + @brief append the object key @a s in quotes, followed by @a tail + + @a tail is the closing quote and the colon, followed by a space when + pretty-printing. The callers pass it because each of them already sits on + the pretty-printing or the compact path; choosing it here would test + @ref pretty_print once more for every key. Most keys are short and plain + and are written in one go by put_short_quoted(); everything else takes the + general path. + */ + template + void put_object_key(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { +#if JSON_LITTLE_ENDIAN_TARGET + if (put_short_quoted(s, tail)) + { + return; + } +#endif + put_char('"'); + dump_escaped(s); + put_literal(tail); + } + + // put_short_quoted() assembles words from little-endian loads +#if JSON_LITTLE_ENDIAN_TARGET + /// the longest string put_short_quoted() takes + static constexpr std::size_t short_string_max = JSON_SCAN_VECTOR ? 32 : 16; + + /// the high bit of every byte of @a v that is not printable ASCII or is a + /// quote, a backslash, or DEL (zero if the bytes can be copied as they are) + static std::uint64_t swar_not_plain(const std::uint64_t v) noexcept + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' + return (((q - ones) & ~q) | ((b - ones) & ~b) // == '"', == '\\' + | ((v - 0x2020202020202020ull) & ~v) // < 0x20 + | v | (v + ones)) // >= 0x7F + & 0x8080808080808080ull; + } + + template + static WordType load_word(const char* p) noexcept + { + WordType w; + std::memcpy(&w, p, sizeof(w)); + return w; + } + + template + static void store_word(char* p, const WordType w) noexcept + { + std::memcpy(p, &w, sizeof(w)); + } + + /*! + @brief write `"`, @a s and @a tail if @a s is short and needs no escaping + + A string of up to 16 (32 with vector instructions) printable ASCII + characters (no quote, backslash, or DEL) is read with two overlapping loads + and written with two overlapping stores, instead of scanning it and calling + memcpy with a length that is only known at run time. Returns false, having + changed nothing, if @a s does not qualify or the write buffer lacks the + room. + */ + template + bool put_short_quoted(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + const std::size_t n = s.size(); + if (n > short_string_max || write_buffer_pos + n + N > write_buffer.size()) + { + return false; + } + + char* d = write_buffer.data() + write_buffer_pos + 1; + const char* const p = s.data(); + + if (n >= 8) + { +#if JSON_SCAN_VECTOR + if (!(n >= 16 ? copy_plain_vectors(d, p, n) : copy_plain_halves(d, p, n))) + { + return false; + } +#else + const auto a = load_word(p); + const auto b = load_word(p + n - 8); + if ((swar_not_plain(a) | swar_not_plain(b)) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 8, b); +#endif + } + else if (n >= 4) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 4); + // the bytes above the string are filled with a plain character + const std::uint64_t w = a | (static_cast(b) << (8 * (n - 4))) + | (0x4141414141414141ull << (8 * n)); + if (swar_not_plain(w) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 4, b); + } + else if (n >= 2) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 2); + const std::uint64_t w = a | (static_cast(b) << (8 * (n - 2))) + | (0x4141414141414141ull << (8 * n)); + if (swar_not_plain(w) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 2, b); + } + else if (n == 1) + { + const std::uint64_t w = static_cast(static_cast(p[0])) | (0x4141414141414141ull << 8u); + if (swar_not_plain(w) != 0) + { + return false; + } + *d = p[0]; + } + + d[-1] = '"'; + std::memcpy(d + n, tail, N - 1); + write_buffer_pos += n + N; + return true; + } + +#if JSON_SCAN_VECTOR + /// copy the 8 to 15 bytes [@a p, @a p + @a n) to @a d with two overlapping + /// words, tested together as one vector, see copy_plain_vectors() + static bool copy_plain_halves(char* d, const char* p, const std::size_t n) noexcept + { +#if JSON_SCAN_NEON + const uint8x8_t lo = vld1_u8(reinterpret_cast(p)); + const uint8x8_t hi = vld1_u8(reinterpret_cast(p + n - 8)); + const uint8x16_t in = vcombine_u8(lo, hi); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)), vceqq_u8(in, vdupq_n_u8(0x7F)))); + if (vmaxvq_u8(special) != 0) + { + return false; + } + vst1_u8(reinterpret_cast(d), lo); + vst1_u8(reinterpret_cast(d + n - 8), hi); +#else + const __m128i lo = _mm_loadl_epi64(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i hi = _mm_loadl_epi64(static_cast(static_cast(p + n - 8))); // NOLINT(bugprone-casting-through-void) + const __m128i in = _mm_unpacklo_epi64(lo, hi); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_or_si128(_mm_cmplt_epi8(in, _mm_set1_epi8(0x20)), _mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)))); + if (_mm_movemask_epi8(special) != 0) + { + return false; + } + _mm_storel_epi64(static_cast<__m128i*>(static_cast(d)), lo); // NOLINT(bugprone-casting-through-void) + _mm_storel_epi64(static_cast<__m128i*>(static_cast(d + n - 8)), hi); // NOLINT(bugprone-casting-through-void) +#endif + return true; + } + + /// copy the 16 to 32 bytes [@a p, @a p + @a n) to @a d with two overlapping + /// vectors, unless one of them is a quote, a backslash, a control + /// character, DEL or not ASCII + static bool copy_plain_vectors(char* d, const char* p, const std::size_t n) noexcept + { +#if JSON_SCAN_NEON + const uint8x16_t a = vld1q_u8(reinterpret_cast(p)); + const uint8x16_t b = vld1q_u8(reinterpret_cast(p + n - 16)); + const uint8x16_t special = vorrq_u8(vorrq_u8(vorrq_u8(vceqq_u8(a, vdupq_n_u8('"')), vceqq_u8(a, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(b, vdupq_n_u8('"')), vceqq_u8(b, vdupq_n_u8('\\')))), + vorrq_u8(vorrq_u8(vcltq_s8(vreinterpretq_s8_u8(a), vdupq_n_s8(0x20)), vcltq_s8(vreinterpretq_s8_u8(b), vdupq_n_s8(0x20))), + vorrq_u8(vceqq_u8(a, vdupq_n_u8(0x7F)), vceqq_u8(b, vdupq_n_u8(0x7F))))); + if (vmaxvq_u8(special) != 0) + { + return false; + } + vst1q_u8(reinterpret_cast(d), a); + vst1q_u8(reinterpret_cast(d + n - 16), b); +#else + const __m128i a = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i b = _mm_loadu_si128(static_cast(static_cast(p + n - 16))); // NOLINT(bugprone-casting-through-void) + const __m128i quote = _mm_set1_epi8('"'); + const __m128i backslash = _mm_set1_epi8('\\'); + const __m128i space = _mm_set1_epi8(0x20); + const __m128i del = _mm_set1_epi8(0x7F); + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(a, quote), _mm_cmpeq_epi8(a, backslash)), + _mm_or_si128(_mm_cmpeq_epi8(b, quote), _mm_cmpeq_epi8(b, backslash))), + _mm_or_si128(_mm_or_si128(_mm_cmplt_epi8(a, space), _mm_cmplt_epi8(b, space)), + _mm_or_si128(_mm_cmpeq_epi8(a, del), _mm_cmpeq_epi8(b, del)))); + if (_mm_movemask_epi8(special) != 0) + { + return false; + } + _mm_storeu_si128(static_cast<__m128i*>(static_cast(d)), a); // NOLINT(bugprone-casting-through-void) + _mm_storeu_si128(static_cast<__m128i*>(static_cast(d + n - 16)), b); // NOLINT(bugprone-casting-through-void) +#endif + return true; + } +#endif +#endif + JSON_PRIVATE_UNLESS_TESTED: /*! @brief dump escaped string @@ -30834,14 +34241,23 @@ class serializer @param[in] s the string to escape Complexity: Linear in the length of string @a s. + + @note kept out of line: with the short-string attempts in front of it gone + from here, compilers would otherwise inline it into @ref put_object_key + and then no longer into the loops that call that */ - void dump_escaped(const string_t& s) + JSON_HEDLEY_NEVER_INLINE void dump_escaped(const string_t& s) { // Short strings of printable ASCII - most object keys and many values - // are checked byte by byte against a table and then copied in one go. // For them, this is cheaper than starting the vector scanner below, // whose setup costs more than such a string takes to scan. The same // check serves both values of ensure_ascii. + // + // Where put_short_quoted() exists and takes strings of up to 32 bytes, + // the callers have tried it already, and repeating the attempt only + // costs time for strings that need escaping or are not ASCII. +#if !JSON_LITTLE_ENDIAN_TARGET || !JSON_SCAN_VECTOR if (s.size() <= 32) { const auto& copyable = ascii_copyable(); @@ -30863,6 +34279,7 @@ class serializer return; } } +#endif // dispatch once here rather than test the flag inside the loop: it does // not change while a string is written, and folding it lets each of the @@ -30940,7 +34357,7 @@ class serializer put_buffer(string_buffer, bytes); bytes = 0; } - put_string(s, i, i + run); + put_run(s, i, run); bytes_after_last_accept = 0; undumped_chars = 0; i += run; @@ -30980,6 +34397,27 @@ class serializer const auto byte = static_cast(s[i]); + if (EnsureAscii && state == UTF8_ACCEPT && byte >= 0x80) + { + // a run of well-formed multi-byte characters is escaped + // straight into the write buffer; an ill-formed sequence + // ends the run and is left to the byte-at-a-time path + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + bytes_after_last_accept = 0; + } + const std::size_t escaped = write_u_escapes(reinterpret_cast(s.data()) + i, s.size() - i); + if (escaped != 0) + { + bytes_after_last_accept = 0; + undumped_chars = 0; + i += escaped - 1; + continue; + } + } + switch (decode(state, codepoint, byte)) { case UTF8_ACCEPT: // decode found a new code point @@ -31257,6 +34695,12 @@ class serializer return; } + // the usual indentation, without a call of memset + if (JSON_HEDLEY_LIKELY(put_short_literal_indent("", indent))) + { + return; + } + const std::size_t capacity = write_buffer.size(); // fill whatever room is left in the buffer; this is the whole job @@ -31290,6 +34734,51 @@ class serializer write_buffer_pos = indent; } + /*! + @brief append a string literal and then @a indent indentation characters + + The usual line break and indentation go into the buffer with one check of + its room and four stores of the indentation, see @ref put_short_literal_indent. + */ + template + JSON_HEDLEY_ALWAYS_INLINE void put_literal_indent(const char (&s)[N], const std::size_t indent) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + if (JSON_HEDLEY_LIKELY(put_short_literal_indent(s, indent))) + { + return; + } + put_literal(s); + put_indent(indent); + } + + /*! + @brief append a string literal and up to 32 indentation characters in one go + + Four words of the indentation character are stored unconditionally and only + the part wanted is claimed, which spares the call of memset. Shared by + @ref put_indent (with an empty literal, whose copy compiles to nothing) and + @ref put_literal_indent. Returns false, having written nothing, if + @a indent is wider or the write buffer lacks the room. + */ + template + JSON_HEDLEY_ALWAYS_INLINE bool put_short_literal_indent(const char (&s)[N], const std::size_t indent) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + constexpr std::size_t length = N - 1; + if (indent > 32 || write_buffer_pos + length + 32 > write_buffer.size()) + { + return false; + } + const std::uint64_t word = 0x0101010101010101ull * static_cast(indent_char); + char* const d = write_buffer.data() + write_buffer_pos; + std::memcpy(d, s, length); + std::memcpy(d + length, &word, sizeof(word)); + std::memcpy(d + length + 8, &word, sizeof(word)); + std::memcpy(d + length + 16, &word, sizeof(word)); + std::memcpy(d + length + 24, &word, sizeof(word)); + write_buffer_pos += length + indent; + return true; + } + /*! @brief append a string literal to the write buffer @@ -31314,6 +34803,28 @@ class serializer write_buffer_pos += length; } + /*! + @brief append the characters of @a str in [@a start, @a start + @a length) + + A run of up to 16 characters with 16 or more in the string from @a start on + and in the write buffer from its end on is copied as a whole block of 16, + which spares the call of memcpy with a length known only at run time; the + surplus characters in the buffer are overwritten by what follows. + */ + template + void put_run(const StringType& str, const std::size_t start, const std::size_t length) + { + // a short run is the exception in text without escapes (CJK, accented + // Latin), for which this keeps the long path the fast one + if (JSON_HEDLEY_UNLIKELY(length <= 16 && start + 16 <= str.size() && write_buffer_pos + 16 <= write_buffer.size())) + { + std::memcpy(write_buffer.data() + write_buffer_pos, str.data() + start, 16); + write_buffer_pos += length; + return; + } + put_string(str, start, start + length); + } + /*! @brief append the characters of @a str in [@a start, @a end) @@ -31556,6 +35067,78 @@ class serializer return escapes; } + /*! + @brief escape the well-formed multi-byte characters at the start of @a data + + Writes the "\\uXXXX" escape of each character (two of them for a character + above U+FFFF) straight into the write buffer, which is why the caller must + have written out what is pending in @ref string_buffer. Stops at the first + byte that is below 0x80 or starts an ill-formed or truncated sequence, which + the caller handles. + + @return the number of bytes of @a data that have been escaped + */ + JSON_HEDLEY_NEVER_INLINE std::size_t write_u_escapes(const unsigned char* data, std::size_t n) + { + constexpr const char* nibble_to_hex = "0123456789abcdef"; + std::size_t i = 0; + while (i < n && data[i] >= 0x80) + { + const std::size_t length = validate_one_utf8(data + i, n - i); + if (length == 0) + { + break; + } + + // a lead byte of 0x80 or above starts a sequence of 2 to 4 bytes + JSON_ASSERT(length >= 2 && length <= 4); + + const auto b0 = static_cast(data[i]); + const auto b1 = static_cast(data[i + 1]); + std::uint32_t codepoint = 0; + switch (length) + { + case 2: + codepoint = ((b0 & 0x1Fu) << 6u) | (b1 & 0x3Fu); + break; + case 3: + codepoint = ((b0 & 0x0Fu) << 12u) | ((b1 & 0x3Fu) << 6u) | (static_cast(data[i + 2]) & 0x3Fu); + break; + default: // 4 + codepoint = ((b0 & 0x07u) << 18u) | ((b1 & 0x3Fu) << 12u) | ((static_cast(data[i + 2]) & 0x3Fu) << 6u) | (static_cast(data[i + 3]) & 0x3Fu); + break; + } + + // at most two escapes of six bytes each + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < 12)) + { + flush(); + } + char* d = write_buffer.data() + write_buffer_pos; + if (codepoint > 0xFFFFu) + { + d = put_u_escape(d, static_cast(0xD7C0u + (codepoint >> 10u)), nibble_to_hex); + codepoint = 0xDC00u + (codepoint & 0x3FFu); + } + d = put_u_escape(d, static_cast(codepoint), nibble_to_hex); + write_buffer_pos = static_cast(d - write_buffer.data()); + i += length; + } + return i; + } + + static char* put_u_escape(char* d, const std::uint16_t codeunit, const char* nibble_to_hex) noexcept + { + const auto unit = static_cast(codeunit); + d[0] = '\\'; + d[1] = 'u'; + d[2] = nibble_to_hex[(unit >> 12u) & 0x0Fu]; + d[3] = nibble_to_hex[(unit >> 8u) & 0x0Fu]; + d[4] = nibble_to_hex[(unit >> 4u) & 0x0Fu]; + d[5] = nibble_to_hex[unit & 0x0Fu]; + return d + 6; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * @@ -31913,11 +35496,22 @@ class serializer /// the output of the serializer (non-owning; the adapter lives at the call site) output_adapter_protocol* o = nullptr; + /// a character buffer left uninitialized: the buffers below are written + /// before they are read (only the part written is ever read), and zeroing + /// 1.6 KiB took more than half of what dumping a small value costs (the + /// user-provided constructor keeps the member initialization, which + /// -Weffc++ asks for, from zeroing them) + template + struct uninitialized_buffer : std::array + { + uninitialized_buffer() noexcept {} // NOLINT(modernize-use-equals-default,hicpp-use-equals-default,cppcoreguidelines-pro-type-member-init,hicpp-member-init): see above + }; + /// a (hopefully) large enough character buffer - std::array number_buffer{{}}; + uninitialized_buffer<64> number_buffer; /// string buffer - std::array string_buffer{{}}; + uninitialized_buffer<512> string_buffer; /// the indentation character const char indent_char; @@ -31937,7 +35531,7 @@ class serializer /// buffer collecting output before it is flushed to the output adapter, so /// that the many small structural writes become few bulk writes static constexpr std::size_t write_buffer_size = 1024; - std::array write_buffer{{}}; + uninitialized_buffer write_buffer; /// number of valid bytes currently held in @ref write_buffer std::size_t write_buffer_pos = 0; }; @@ -32468,9 +36062,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend class ::nlohmann::detail::binary_writer; template friend class ::nlohmann::detail::binary_reader; - template + template friend class ::nlohmann::detail::json_sax_dom_parser; template + friend class ::nlohmann::detail::json_sax_dom_binary_parser; + template friend class ::nlohmann::detail::json_sax_dom_callback_parser; #if JSON_DIAGNOSTIC_POSITIONS friend struct ::nlohmann::detail::diagnostic_positions; @@ -38597,7 +42193,7 @@ public: from_ubjson/from_bjdata/from_bon8/from_bson overloads Building the @ref binary_reader as a named local, rather than as a - temporary that @ref detail::json_sax_dom_parser::parse is called on in + temporary that @ref detail::json_sax_dom_binary_parser::parse is called on in the same expression, avoids a false-positive cppcheck accessMoved warning in each of the 16 callers. */ @@ -38608,7 +42204,7 @@ public: const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) { basic_json result; - detail::json_sax_dom_parser sdp(result, allow_exceptions); + detail::json_sax_dom_binary_parser sdp(result, allow_exceptions); binary_reader reader(std::move(ia), format, error_handler); if (!reader.sax_parse(&sdp, strict, tag_handler)) { @@ -38631,7 +42227,7 @@ public: const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_cbor(j); return result; } @@ -38641,7 +42237,9 @@ public: static void to_cbor(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_cbor(j); + binary_writer writer(o, error_handler); + writer.write_cbor(j); + writer.flush(); } /// @brief create a CBOR serialization of a given JSON value @@ -38649,7 +42247,9 @@ public: static void to_cbor(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_cbor(j); + binary_writer writer(o, error_handler); + writer.write_cbor(j); + writer.flush(); } /// @brief create a MessagePack serialization of a given JSON value @@ -38658,7 +42258,7 @@ public: const error_handler_t error_handler = error_handler_t::keep) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_msgpack(j); return result; } @@ -38668,7 +42268,9 @@ public: static void to_msgpack(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = error_handler_t::keep) { - binary_writer(o, error_handler).write_msgpack(j); + binary_writer writer(o, error_handler); + writer.write_msgpack(j); + writer.flush(); } /// @brief create a MessagePack serialization of a given JSON value @@ -38676,7 +42278,9 @@ public: static void to_msgpack(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = error_handler_t::keep) { - binary_writer(o, error_handler).write_msgpack(j); + binary_writer writer(o, error_handler); + writer.write_msgpack(j); + writer.flush(); } /// @brief create a UBJSON serialization of a given JSON value @@ -38687,7 +42291,7 @@ public: const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_ubjson(j, use_size, use_type); return result; } @@ -38698,7 +42302,9 @@ public: const bool use_size = false, const bool use_type = false, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_ubjson(j, use_size, use_type); + binary_writer writer(o, error_handler); + writer.write_ubjson(j, use_size, use_type); + writer.flush(); } /// @brief create a UBJSON serialization of a given JSON value @@ -38707,7 +42313,9 @@ public: const bool use_size = false, const bool use_type = false, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_ubjson(j, use_size, use_type); + binary_writer writer(o, error_handler); + writer.write_ubjson(j, use_size, use_type); + writer.flush(); } /// @brief create a BJData serialization of a given JSON value @@ -38719,7 +42327,7 @@ public: const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -38731,7 +42339,9 @@ public: const bjdata_version_t version = bjdata_version_t::draft2, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_ubjson(j, use_size, use_type, true, true, version); + binary_writer writer(o, error_handler); + writer.write_ubjson(j, use_size, use_type, true, true, version); + writer.flush(); } /// @brief create a BJData serialization of a given JSON value @@ -38741,7 +42351,9 @@ public: const bjdata_version_t version = bjdata_version_t::draft2, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_ubjson(j, use_size, use_type, true, true, version); + binary_writer writer(o, error_handler); + writer.write_ubjson(j, use_size, use_type, true, true, version); + writer.flush(); } /// @brief create a BSON serialization of a given JSON value @@ -38750,7 +42362,7 @@ public: const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result, error_handler).write_bson(j); return result; } @@ -38760,7 +42372,9 @@ public: static void to_bson(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_bson(j); + binary_writer writer(o, error_handler); + writer.write_bson(j); + writer.flush(); } /// @brief create a BSON serialization of a given JSON value @@ -38768,7 +42382,9 @@ public: static void to_bson(const basic_json& j, detail::output_adapter o, const error_handler_t error_handler = detail::binary_writer_default_error_handler()) { - binary_writer(o, error_handler).write_bson(j); + binary_writer writer(o, error_handler); + writer.write_bson(j); + writer.flush(); } /// @brief create a BON8 serialization of a given JSON value @@ -38776,7 +42392,7 @@ public: static std::vector to_bon8(const basic_json& j) { std::vector result; - result.reserve(detail::binary_reserve_hint(j)); + result.reserve(detail::binary_initial_capacity(j)); vector_writer(result).write_bon8(j); return result; } @@ -38785,14 +42401,18 @@ public: /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ static void to_bon8(const basic_json& j, detail::output_adapter o) { - binary_writer(o).write_bon8(j); + binary_writer writer(o); + writer.write_bon8(j); + writer.flush(); } /// @brief create a BON8 serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bon8/ static void to_bon8(const basic_json& j, detail::output_adapter o) { - binary_writer(o).write_bon8(j); + binary_writer writer(o); + writer.write_bon8(j); + writer.flush(); } /// @brief create a JSON value from an input in CBOR format @@ -39188,6 +42808,72 @@ public: /// @name JSON Patch functions /// @{ + private: + /// the members of an operation of a JSON patch that are of interest + struct patch_members + { + basic_json* op = nullptr; + basic_json* path = nullptr; + basic_json* value = nullptr; + basic_json* from = nullptr; + }; + + /// @brief find the members "op", "path", "value", and "from" of the object @a val in one pass + static patch_members find_patch_members(const basic_json& val, std::true_type /*key_type is string_t*/) + { + patch_members members; + for (auto& member : *val.m_data.m_value.object) + { + const string_t& key = member.first; + switch (key.size()) + { + case 2: + if (key == "op") + { + members.op = &member.second; + } + break; + case 4: + if (key == "path") + { + members.path = &member.second; + } + else if (key == "from") + { + members.from = &member.second; + } + break; + case 5: + if (key == "value") + { + members.value = &member.second; + } + break; + default: + break; + } + } + return members; + } + + /// @brief find the members by lookup for key types other than string_t + static patch_members find_patch_members(const basic_json& val, std::false_type /*key_type is not string_t*/) + { + const auto find_member = [&val](const char* name) -> basic_json * + { + const auto it = val.m_data.m_value.object->find(string_t(name)); + return it != val.m_data.m_value.object->end() ? &it->second : nullptr; + }; + + patch_members members; + members.op = find_member("op"); + members.path = find_member("path"); + members.value = find_member("value"); + members.from = find_member("from"); + return members; + } + + public: /// @brief applies a JSON patch in-place without copying the object /// @sa https://json.nlohmann.me/api/basic_json/patch/ void patch_inplace(const basic_json& json_patch) @@ -39361,18 +43047,18 @@ public: // a diff creates them: apply the operation and return true; return // false, with nothing changed, for all others string_t token; - const string_t value_key = "value"; - const auto try_plain = [&result, &token, &value_key](patch_operations operation, const string_t& path, const basic_json & val) + typename json_pointer::template plain_steps steps; + const auto try_plain = [&result, &token, &steps](patch_operations operation, const string_t& path, const patch_members & members, const basic_json & val) { - if (path.empty() || path.find_first_of('~') != string_t::npos) + if (path.empty()) { return false; } // a missing "value" is reported below - const auto value_it = val.m_data.m_value.object->find(value_key); - const basic_json& value = value_it != val.m_data.m_value.object->end() ? value_it->second : val; - if (operation != patch_operations::remove && value_it == val.m_data.m_value.object->end()) + const basic_json* const value_ptr = members.value; + const basic_json& value = value_ptr != nullptr ? *value_ptr : val; + if (operation != patch_operations::remove && value_ptr == nullptr) { return false; } @@ -39381,7 +43067,7 @@ public: { case patch_operations::replace: { - basic_json* const target = json_pointer::resolve_plain(&result, path, path.size(), token); + basic_json* const target = json_pointer::resolve_plain(&result, path, path.size(), token, steps); if (target == nullptr) { return false; @@ -39392,7 +43078,7 @@ public: case patch_operations::test: { - const basic_json* const target = json_pointer::resolve_plain(&result, path, path.size(), token); + const basic_json* const target = json_pointer::resolve_plain(&result, path, path.size(), token, steps); // a failing test is reported below return target != nullptr && *target == value; } @@ -39412,13 +43098,17 @@ public: } --slash; - basic_json* const parent = json_pointer::resolve_plain(&result, path, slash, token); + basic_json* const parent = json_pointer::resolve_plain(&result, path, slash, token, steps); if (parent == nullptr) { return false; } token.clear(); token.append(path.data() + slash + 1, path.size() - slash - 1); + if (token.find_first_of('~') != string_t::npos) + { + return false; + } if (operation == patch_operations::add) { @@ -39474,31 +43164,32 @@ public: } }; + // the name of the first member to check + const string_t op_member = "op"; + // iterate and apply the operations for (const auto& val : json_patch) { // wrapper to get a value for an operation - const auto get_value = [&val](const string_t& op, - const string_t& member, + const auto get_value = [&val](basic_json * found, + const string_t& op, + const char* member, bool string_type) -> basic_json & { - // find value - auto it = val.m_data.m_value.object->find(member); - // check if the desired value is present - if (JSON_HEDLEY_UNLIKELY(it == val.m_data.m_value.object->end())) + if (JSON_HEDLEY_UNLIKELY(found == nullptr)) { - detail::throw_patch_missing_member(op, member, false, val); + detail::throw_patch_missing_member(op, string_t(member), false, val); } // check if the result is of type string - if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) + if (JSON_HEDLEY_UNLIKELY(string_type && !found->is_string())) { - detail::throw_patch_missing_member(op, member, true, val); + detail::throw_patch_missing_member(op, string_t(member), true, val); } // no error: return value - return it->second; + return *found; }; // type check: every element of the array must be an object @@ -39507,26 +43198,30 @@ public: detail::throw_patch_not_an_array(val); } + const patch_members members = find_patch_members(val, std::is_same {}); + // collect mandatory members - const string_t& op = *get_value("op", "op", true).m_data.m_value.string; - const string_t& path = *get_value(op, "path", true).m_data.m_value.string; + const string_t& op = *get_value(members.op, op_member, "op", true).m_data.m_value.string; + const string_t& path = *get_value(members.path, op, "path", true).m_data.m_value.string; const patch_operations operation = get_op(op); // the usual operations on a plain pointer are applied without // building the pointer's reference tokens; anything else (and // every error) is handled below - if (try_plain(operation, path, val)) + if (try_plain(operation, path, members, val)) { continue; } + // the general operations may change any container + steps.reset(); json_pointer ptr(path); switch (operation) { case patch_operations::add: { - operation_add(ptr, get_value("add", "value", false)); + operation_add(ptr, get_value(members.value, "add", "value", false)); break; } @@ -39539,13 +43234,13 @@ public: case patch_operations::replace: { // the "path" location must exist - use at() - result.at(ptr) = get_value("replace", "value", false); + result.at(ptr) = get_value(members.value, "replace", "value", false); break; } case patch_operations::move: { - const string_t& from_path = *get_value("move", "from", true).m_data.m_value.string; + const string_t& from_path = *get_value(members.from, "move", "from", true).m_data.m_value.string; json_pointer from_ptr(from_path); if (JSON_HEDLEY_UNLIKELY(is_proper_prefix(from_ptr, ptr))) @@ -39568,7 +43263,7 @@ public: case patch_operations::copy: { - const string_t& from_path = *get_value("copy", "from", true).m_data.m_value.string; + const string_t& from_path = *get_value(members.from, "copy", "from", true).m_data.m_value.string; const json_pointer from_ptr(from_path); // The copy is functionally identical to an "add" @@ -39586,7 +43281,7 @@ public: { // check if "value" matches the one at "path" // the "path" location must exist - use at() - success = (result.at(ptr) == get_value("test", "value", false)); + success = (result.at(ptr) == get_value(members.value, "test", "value", false)); } JSON_INTERNAL_CATCH (out_of_range&) { @@ -39678,13 +43373,18 @@ public: /// @brief append an operation @a op for @a path to @a result, with @a value if it is not null static void diff_operation(basic_json& result, const char* op, const string_t& path, const basic_json* value) { + // the keys are distinct and in the order of std::map; the members are + // added without looking them up basic_json entry(value_t::object); - entry["op"] = op; - entry["path"] = path; + auto& members = *entry.m_data.m_value.object; + auto hint = members.end(); + json_pointer::add_flattened(members, hint, string_t("op"), string_t(op), detail::priority_tag<2> {}); + json_pointer::add_flattened(members, hint, string_t("path"), path, detail::priority_tag<2> {}); if (value != nullptr) { - entry["value"] = *value; + json_pointer::add_flattened(members, hint, string_t("value"), *value, detail::priority_tag<2> {}); } + entry.set_parents(); result.push_back(std::move(entry)); } @@ -39706,6 +43406,62 @@ public: diff_operation(result, "add", path, &value); } + /// @brief the values in the target of the members of a source object, for the + /// objects of up to 16 members without allocating + /// + /// reserve() must be called with the number of values that are added. + class diff_target_values + { + public: + diff_target_values() noexcept + : inline_items() // NOLINT(readability-redundant-member-init) + {} + diff_target_values(const diff_target_values&) = delete; + diff_target_values& operator=(const diff_target_values&) = delete; + diff_target_values(diff_target_values&&) = delete; + diff_target_values& operator=(diff_target_values&&) = delete; + ~diff_target_values() = default; + + void reserve(const std::size_t n) + { + if (n > inline_size) + { + large.resize(n); + items = large.data(); + } + } + + void push_back(const basic_json* value) + { + items[count++] = value; + } + + std::size_t size() const noexcept + { + return count; + } + + const basic_json* operator[](const std::size_t i) const noexcept + { + return items[i]; + } + + private: + static constexpr std::size_t inline_size = 16; + /// the values of a small object, left uninitialized: they are written + /// before they are read (the user-provided constructor keeps the + /// member initialization, which -Weffc++ asks for, from zeroing them) + struct uninitialized_items : std::array + { + uninitialized_items() noexcept {} // NOLINT(modernize-use-equals-default,hicpp-use-equals-default,cppcoreguidelines-pro-type-member-init,hicpp-member-init): see above + }; + + std::size_t count = 0; + std::vector large{}; // NOLINT(readability-redundant-member-init) + uninitialized_items inline_items; + const basic_json** items = inline_items.data(); + }; + /// @brief append "/" and the decimal representation of @a index to @a path static void diff_append_index(string_t& path, std::size_t index) { @@ -39786,8 +43542,9 @@ public: operations that remove every member of @a source and add every member of @a target to @a result, and returns false. */ + template static bool diff_object_keys(basic_json& result, const basic_json& source, const basic_json& target, - const string_t& path, std::vector& target_values, + const string_t& path, TargetValues& target_values, std::vector& added) { // first pass: one lookup per source key @@ -40001,7 +43758,7 @@ public: case value_t::object: { - std::vector target_values; + diff_target_values target_values; std::vector added; if (diff_object_keys(result, source, target, path, target_values, added)) { diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 6220270c7..6de8ee77d 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -30,6 +30,7 @@ #include // array #include // nullptr_t, size_t // IWYU pragma: keep #include // uint8_t, uint32_t +#include // FILE #include // memcpy, strlen #include // istream // IWYU pragma: keep #include // distance, input_iterator_tag, iterator_traits @@ -756,6 +757,34 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned } return p; } + +/*! +@brief copy 16 bytes from p to w, and find the first quote, backslash, +control character, or non-ASCII byte among them + +The bytes are copied whether or not they are all plain (the caller has room +for 16 bytes and keeps only the plain ones). + +@return the offset of that byte, or 16 if there is none +*/ +NLOHMANN_VIEW_ALWAYS_INLINE unsigned copy_plain_run16(const unsigned char* p, char* w) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t in = vld1q_u8(p); + vst1q_u8(reinterpret_cast(w), in); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); + return bits != 0 ? static_cast(count_trailing_zeros(bits) >> 2u) : 16u; +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + _mm_storeu_si128(static_cast<__m128i*>(static_cast(w)), in); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); + return bits != 0 ? static_cast(count_trailing_zeros(bits)) : 16u; +#endif +} #endif #if NLOHMANN_VIEW_SSSE3_DISPATCH @@ -1437,11 +1466,34 @@ class builder 0, 0, '\b', 0, 0, 0, '\f', 0, 0, 0, 0, 0, 0, 0, '\n', 0, 0, 0, '\r', 0, '\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + // The write position is kept in locals: the stores of characters into + // the arena may alias the members of the builder, which would then be + // reloaded after every one. It is stored back where the arena grows, + // and by `write_back` on every return. + const unsigned char* const end = m_end; const std::size_t start = arena_used(); - arena_run(s, static_cast(p - s)); + char* w = m_arena_write; + const char* w_end = m_arena_end; + const arena_write_back write_back(*this, w); + { + const auto n = static_cast(p - s); + if (n <= 16 && end - s >= 16) + { + arena_ensure(w, w_end, 16); + std::memcpy(w, s, 16); + } + else if (n != 0) + { + // (an empty run leaves an empty arena unallocated, and memcpy() + // must not be passed its null pointer even for zero bytes) + arena_ensure(w, w_end, n); + std::memcpy(w, s, n); + } + w += n; + } for (;;) { - if (p == m_end) + if (p == end) { return failed(error_code::string_missing_quote, p); } @@ -1449,7 +1501,9 @@ class builder if (c == '"') { ++p; - return decoded{p, start, arena_used() - start}; + // (from w: `write_back` stores it only after the return value + // is made; the arena is allocated, since an escape was written) + return decoded{p, start, static_cast(w - m_doc.arena.data()) - start}; } if (c != '\\') { @@ -1459,7 +1513,7 @@ class builder return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); } ++p; - if (p == m_end) + if (p == end) { return failed(error_code::string_missing_quote, p); } @@ -1477,7 +1531,7 @@ class builder { return failed(error_code::string_surrogate_low, p); } - if (m_end - p < 2 || p[0] != '\\' || p[1] != 'u') + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') { return failed(error_code::string_surrogate_high, p); } @@ -1493,26 +1547,106 @@ class builder } cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); } - arena_ensure(4); - m_arena_write = put_utf8(m_arena_write, cp); + arena_ensure(w, w_end, 4); + w = put_utf8(w, cp); } else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0)) { - arena_ensure(1); - *m_arena_write++ = simple_escape[d]; + arena_ensure(w, w_end, 1); + *w++ = simple_escape[d]; } else { --p; return failed(error_code::string_escape, p); } - if (p != m_end && *p == '\\') +#if NLOHMANN_VIEW_NEON + if (p != end && *p == '\\') { continue; // consecutive escapes ("\u00e4\u00f6"): no run in between } + // The first 16 bytes of the run are checked one by one, in predicted branches, and copied + // as one block: where the runs between escapes are short, the vector step below would put + // its longer dependency chain (load, compare, mask, count, add) on p for each of them, + // and be slower than these steps (the scanner of the keys works the same way) + { + if (end - p >= 16) + { + const std::uint8_t* const plain = string_plain(); + std::size_t steps = 16; +#define NLOHMANN_VIEW_STEP(i) if (NLOHMANN_VIEW_LIKELY(plain[p[i]] != 0)) {} else { steps = (i); goto run_found; } + NLOHMANN_VIEW_REPEAT16(NLOHMANN_VIEW_STEP) +#undef NLOHMANN_VIEW_STEP +run_found: + arena_ensure(w, w_end, 16); + std::memcpy(w, p, 16); // (only the first `steps` bytes are kept) + w += steps; + p += steps; + if (steps < 16) + { + if (*p >= 0x80) + { + // non-ASCII: the UTF-8 check, and the rest of the run + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + arena_ensure(w, w_end, n); + std::memcpy(w, r, n); + w += n; + } + continue; + } + } + } +#else + if (p != end && (*p == '\\' || *p == '"')) + { + continue; // consecutive escapes ("\u00e4\u00f6"), or the end of the string: no run in between + } +#endif +#if NLOHMANN_VIEW_VECTOR + // the plain bytes up to the next quote, backslash, or control + // character are copied as they are scanned, 16 bytes at a time + for (;;) + { + if (NLOHMANN_VIEW_UNLIKELY(end - p < 16)) + { + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + arena_ensure(w, w_end, n); + std::memcpy(w, r, n); + w += n; + break; + } + arena_ensure(w, w_end, 16); + const unsigned k = copy_plain_run16(p, w); + w += k; + p += k; + if (k == 16) + { + continue; + } + if (NLOHMANN_VIEW_UNLIKELY(*p >= 0x80)) + { + // non-ASCII: the UTF-8 check, and the rest of the run + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + arena_ensure(w, w_end, n); + std::memcpy(w, r, n); + w += n; + } + break; + } +#else const unsigned char* const r = p; - p = scan_string_run(p, m_end); - arena_run(r, static_cast(p - r)); + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + arena_ensure(w, w_end, n); + std::memcpy(w, r, n); + w += n; +#endif } } @@ -1554,19 +1688,49 @@ class builder char* m_arena_write = nullptr; char* m_arena_end = nullptr; + /// room for n more bytes at w, the write position kept in a local, with + /// w_end the end of the arena. (A lambda instead is not always inlined, + /// and then w would be kept in memory, as m_arena_write is.) + NLOHMANN_VIEW_ALWAYS_INLINE void arena_ensure(char*& w, const char*& w_end, std::size_t n) + { + if (NLOHMANN_VIEW_UNLIKELY(static_cast(w_end - w) < n)) + { + m_arena_write = w; + arena_grow(n); + w = m_arena_write; + w_end = m_arena_end; + } + } + + /// stores a write position kept in a local back into m_arena_write when + /// the function that keeps it returns, on every path + class arena_write_back + { + public: + arena_write_back(builder& b, char*& w) noexcept + : m_builder(b) + , m_w(w) + {} + ~arena_write_back() + { + m_builder.m_arena_write = m_w; + } + + arena_write_back(const arena_write_back&) = delete; + arena_write_back& operator=(const arena_write_back&) = delete; + arena_write_back(arena_write_back&&) = delete; + arena_write_back& operator=(arena_write_back&&) = delete; + + private: + builder& m_builder; + char*& m_w; + }; + NLOHMANN_VIEW_ALWAYS_INLINE std::size_t arena_used() const noexcept { return m_arena_write != nullptr ? static_cast(m_arena_write - m_doc.arena.data()) : 0; } - NLOHMANN_VIEW_ALWAYS_INLINE void arena_ensure(std::size_t n) - { - if (NLOHMANN_VIEW_UNLIKELY(static_cast(m_arena_end - m_arena_write) < n)) - { - arena_grow(n); - } - } - NLOHMANN_VIEW_NOINLINE void arena_grow(std::size_t n) { const std::size_t used = arena_used(); @@ -1575,25 +1739,6 @@ class builder m_arena_end = m_doc.arena.data() + m_doc.arena.size(); } - /// append the run [r, r + n) to the arena; short runs as one fixed-size - /// 16-byte move when both sides have the room (no library call) - NLOHMANN_VIEW_ALWAYS_INLINE void arena_run(const unsigned char* r, std::size_t n) - { - if (n <= 16 && m_end - r >= 16) - { - arena_ensure(16); - std::memcpy(m_arena_write, r, 16); - } - else if (n != 0) - { - // (an empty run leaves an empty arena unallocated, and memcpy() - // must not be passed its null pointer even for zero bytes) - arena_ensure(n); - std::memcpy(m_arena_write, r, n); - } - m_arena_write += n; - } - /// UTF-8 encoding of cp at w (room for 4 bytes); returns the position after /// it. (A returned pointer stays in a register; writing through a /// reference to m_arena_write would store and reload it around every byte.) @@ -2116,11 +2261,11 @@ indent_done: ++p; } const unsigned char* const int_start = p; - if (p != e && *p == '0') + if (cur() == '0') { ++p; } - else if (NLOHMANN_VIEW_LIKELY(p != e && *p >= '1' && *p <= '9')) + else if (NLOHMANN_VIEW_LIKELY(cur() >= '1' && cur() <= '9')) { p = skip_digits(p + 1, e); } @@ -2131,7 +2276,7 @@ indent_done: const auto int_digits = static_cast(p - int_start); std::size_t frac_digits = 0; bool is_float = false; - if (p != e && *p == '.') + if (cur() == '.') { ++p; const unsigned char* const f0 = p; @@ -2144,20 +2289,20 @@ indent_done: is_float = true; } std::int64_t exponent = 0; - if (p != e && (*p | 0x20) == 'e') + if ((cur() | 0x20) == 'e') { ++p; bool exp_negative = false; - if (p != e && (*p == '+' || *p == '-')) + if (cur() == '+' || cur() == '-') { exp_negative = *p == '-'; ++p; } - if (NLOHMANN_VIEW_UNLIKELY(p == e || !is_digit(*p))) + if (NLOHMANN_VIEW_UNLIKELY(!is_digit(cur()))) { return fail(error_code::number_after_exponent); } - while (p != e && is_digit(*p)) + while (is_digit(cur())) { if (exponent < 100000) { @@ -2219,7 +2364,7 @@ indent_done: ++p; // opening quote const unsigned char* const s = p; p = scan_string_run(p, e); - if (NLOHMANN_VIEW_LIKELY(p != e && *p == '"')) + if (NLOHMANN_VIEW_LIKELY(cur() == '"')) { push_node(value_t::string, 0, 0, static_cast(s - b), static_cast(p - s)); ++p; @@ -3168,27 +3313,96 @@ constexpr std::size_t index_max_displacement = 64; static_assert(document_data::index_min_members >= 2 * index_max_displacement, "a table must be much larger than the probe window of a lookup"); -/// hash of a key: its bytes, eight at a time, in a fixed byte order +/// hash of a key: its bytes, eight at a time (in a fixed byte order), where the +/// last word may overlap the one before (so that no loop over single bytes is +/// needed); a key of fewer than eight bytes is read as two overlapping words of +/// four or two bytes (in the native byte order: the hash is not stored, and the +/// same on both sides of a lookup) inline std::uint64_t key_hash(const char* s, std::size_t n) noexcept { std::uint64_t h = 0x9E3779B97F4A7C15u * (n + 1); const auto* p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - while (n >= 8) - { - h = (h ^ read_eight_bytes(p)) * 0xBF58476D1CE4E5B9u; - h ^= h >> 29u; - p += 8; - n -= 8; - } std::uint64_t w = 0; - for (std::size_t i = 0; i < n; ++i) + if (n >= 8) { - w |= static_cast(p[i]) << (8u * i); + const unsigned char* const last = p + n - 8; + while (p < last) + { + h = (h ^ read_eight_bytes(p)) * 0xBF58476D1CE4E5B9u; + h ^= h >> 29u; + p += 8; + } + w = read_eight_bytes(last); } - h = (h ^ w) * 0x94D049BB133111EBu; + else if (n >= 4) + { + std::uint32_t head = 0; + std::uint32_t tail = 0; + std::memcpy(&head, p, 4); + std::memcpy(&tail, p + n - 4, 4); + w = static_cast(head) | (static_cast(tail) << 32u); + } + else if (n >= 2) + { + std::uint16_t head = 0; + std::uint16_t tail = 0; + std::memcpy(&head, p, 2); + std::memcpy(&tail, p + n - 2, 2); + w = static_cast(head) | (static_cast(tail) << 16u); + } + else if (n == 1) + { + w = p[0]; + } + // (a product only carries bits upwards, and the slot is taken from the low + // bits: the high bits of the last word are folded down before the multiplication) + h = (h ^ w ^ (w >> 29u)) * 0x94D049BB133111EBu; return h ^ (h >> 31u); } +/// whether the n <= 16 bytes at a and b are equal, as two overlapping words +/// (the first and the last 8, 4, or 2 bytes) and without a call +NLOHMANN_VIEW_ALWAYS_INLINE bool same_short_bytes(const unsigned char* a, const unsigned char* b, std::size_t n) noexcept +{ + if (n >= 8) + { + std::uint64_t x0 = 0; + std::uint64_t x1 = 0; + std::uint64_t y0 = 0; + std::uint64_t y1 = 0; + std::memcpy(&x0, a, 8); + std::memcpy(&x1, a + n - 8, 8); + std::memcpy(&y0, b, 8); + std::memcpy(&y1, b + n - 8, 8); + return x0 == y0 && x1 == y1; + } + if (n >= 4) + { + std::uint32_t x0 = 0; + std::uint32_t x1 = 0; + std::uint32_t y0 = 0; + std::uint32_t y1 = 0; + std::memcpy(&x0, a, 4); + std::memcpy(&x1, a + n - 4, 4); + std::memcpy(&y0, b, 4); + std::memcpy(&y1, b + n - 4, 4); + return x0 == y0 && x1 == y1; + } + if (n >= 2) + { + std::uint16_t x0 = 0; + std::uint16_t x1 = 0; + std::uint16_t y0 = 0; + std::uint16_t y1 = 0; + std::memcpy(&x0, a, 2); + std::memcpy(&x1, a + n - 2, 2); + std::memcpy(&y0, b, 2); + std::memcpy(&y1, b + n - 2, 2); + return x0 == y0 && x1 == y1; + } + return n == 0 || *a == *b; +} + /// build the table of a large object inline void build_object_index(document_data& d, node* obj) { @@ -3253,6 +3467,27 @@ inline const node* find_indexed(const document_data& d, const node* obj, const c const document_data::object_index& ix = d.indexes[obj->extra - 1u]; const std::uint32_t* const table = d.index_slots.data() + ix.start; std::size_t slot = conditional_static_cast(key_hash(key, n)) & ix.mask; + const auto* const k_bytes = reinterpret_cast(key); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + // (the loop for longer keys comes first, as the code is laid out; it is not the more common case) + if (NLOHMANN_VIEW_UNLIKELY(n <= 16)) + { + // (up to 16 bytes, the bytes are compared as two words, without a call) + for (std::size_t distance = 0; distance <= index_max_displacement; ++distance) + { + const std::uint32_t s = table[slot]; + if (s == 0) + { + return nullptr; + } + const node* const k = obj + s; + if (k->len == n && same_short_bytes(reinterpret_cast(d.str(*k)), k_bytes, n)) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + { + return k; + } + slot = (slot + 1) & ix.mask; + } + return nullptr; + } for (std::size_t distance = 0; distance <= index_max_displacement; ++distance) { const std::uint32_t s = table[slot]; @@ -3421,6 +3656,10 @@ const node* element_at(const document_data& d, const node* array, std::size_t id { return e + idx; // a growable block: one link per element } + if (array->next == array->len + 1u && (!nav::editable() || (array->flags & node_flags::moved) == 0)) + { + return e + idx; // no element has a subtree: the elements are the nodes after the array + } for (std::size_t i = 0; i < idx; ++i) { e = document_data::after(e); @@ -3436,6 +3675,10 @@ const node* last_child(const document_data& d, const node* container) noexcept const std::size_t value_offset = container->kind == static_cast(value_t::object) ? 1 : 0; const node* const end = navigation::end(d, container); const node* last = navigation::first(d, container); + if (value_offset == 0 && container->next == container->len + 1u && (!navigation::editable() || (container->flags & node_flags::moved) == 0)) + { + return last + (container->len - 1u); // an array without subtrees: the elements are the nodes after it + } for (const node* c = document_data::after(last + value_offset); c != end; c = document_data::after(c + value_offset)) { last = c; @@ -5301,6 +5544,9 @@ NLOHMANN_JSON_NAMESPACE_END #include // max #include // size_t +#ifndef JSON_NO_IO + #include // FILE, fread +#endif #include // istream, streambuf #include // basic_string, char_traits, string #include // decay, integral_constant, is_array, is_const, is_integral, is_lvalue_reference, is_pointer, is_same, remove_cv, remove_extent, remove_pointer, remove_reference @@ -5446,6 +5692,37 @@ inline void append_stream(std::string& buf, std::istream& is) is.clear(is.rdstate() | std::ios::eofbit); } +/// a pointer to a C stream (never with JSON_NO_IO, which leaves out ) +template +struct is_c_stream +#ifndef JSON_NO_IO + : std::is_same +#else + : std::false_type +#endif +{}; + +#ifndef JSON_NO_IO +/// append the rest of a C stream: in blocks with fread, not character by +/// character (the library's file adapter reads with fgetc); +/// f is not null +inline void append_file(std::string& buf, std::FILE* f) +{ + for (;;) + { + const std::size_t used = buf.size(); + const std::size_t room = (std::max)(buf.capacity() - used, std::size_t{4096}); + buf.resize(used + room); + const std::size_t got = std::fread(&buf[used], 1, room, f); + buf.resize(used + got); + if (got < room) + { + break; // end of file, or an error (as fgetc, which returns EOF for both) + } + } +} +#endif + } // namespace view } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -5604,20 +5881,21 @@ class view_item view_item(const iterator& it, std::size_t index) : m_it(it) - { - if (!it.is_object_iterator()) - { - m_index = std::to_string(index); - } - } + , m_position(index) + {} - /// the member key, or the element index for arrays + /// the member key, or the element index for arrays (as text made on the + /// first call, so that iterating over the values pays nothing for it) string_view_t key() const { if (m_it.is_object_iterator()) { return m_it.key(); } + if (m_index.empty()) + { + m_index = std::to_string(m_position); + } return string_view_t(m_index.data(), m_index.size()); } @@ -5640,7 +5918,8 @@ class view_item private: iterator m_it; - std::string m_index{}; // NOLINT(readability-redundant-member-init) + std::size_t m_position; ///< the index of an array element + mutable std::string m_index{}; ///< the text of m_position, once key() was called // NOLINT(readability-redundant-member-init) }; /// the range returned by basic_json_view::items() @@ -5734,7 +6013,7 @@ NLOHMANN_JSON_NAMESPACE_END -#include // max +#include // array #include // size_t #include // int64_t, uint8_t #include // string @@ -5770,12 +6049,13 @@ template BasicJsonType materialize(const document_data& d, const node* n) { using string_t = typename BasicJsonType::string_t; - using sax_t = json_sax_dom_parser>; + using sax_t = json_sax_dom_parser, true>; using nav = navigation; // the open containers: their next element (or key of the next member), - // their end, and whether they are objects; sized once from the - // document's deepest nesting (for a subtree an upper bound) + // their end, and whether they are objects; for the first 16 levels on the + // stack, else sized once from the document's deepest nesting (for a + // subtree an upper bound) struct frame { const node* pos; @@ -5783,10 +6063,43 @@ BasicJsonType materialize(const document_data& d, const node* n) bool object; }; + // a scalar needs neither the handler nor the stack of open containers + switch (static_cast(n->kind)) + { + case value_t::string: + return BasicJsonType(string_t(d.str(*n), n->len)); + case value_t::number_integer: + return BasicJsonType(static_cast(static_cast(integer_bits(*n)))); + case value_t::number_unsigned: + return BasicJsonType(static_cast(integer_bits(*n))); + case value_t::number_float: + return BasicJsonType(float_value(d, *n)); + case value_t::boolean: + return BasicJsonType((n->flags & node_flags::is_true) != 0); + case value_t::null: + return BasicJsonType(nullptr); + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + BasicJsonType result; sax_t sax(result, true); const string_t no_token{}; - std::vector open((std::max)(d.max_depth, std::size_t{1})); + // (shallow subtrees on the stack: a call for a small value then allocates only for the value) + std::array shallow; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled as containers open + std::vector deep{}; + frame* open = shallow.data(); + std::size_t capacity = shallow.size(); + if (d.max_depth > capacity) + { + deep.resize(d.max_depth); + open = deep.data(); + capacity = deep.size(); + } std::size_t depth = 0; for (;;) { @@ -5806,9 +6119,13 @@ BasicJsonType materialize(const document_data& d, const node* n) { sax.start_array(n->len); } - if (NLOHMANN_VIEW_UNLIKELY(depth == open.size())) + if (NLOHMANN_VIEW_UNLIKELY(depth == capacity)) { - open.resize(depth * 2); // (deeper than the recorded nesting, e.g. after edits) + // (deeper than the recorded nesting, e.g. after edits) + deep.assign(open, open + depth); + deep.resize(depth * 2); + open = deep.data(); + capacity = deep.size(); } open[depth++] = frame{nav::first(d, n), nav::end(d, n), object}; break; @@ -5835,6 +6152,8 @@ BasicJsonType materialize(const document_data& d, const node* n) case value_t::binary: case value_t::discarded: default: + // false positive: the parser's frame stack points into its own storage (never null) + // @infer-ignore NULLPTR_DEREFERENCE sax.null(); break; } @@ -6089,6 +6408,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -6146,13 +6467,6 @@ class output_buffer put(s, N - 1); } - void put_repeated(char c, std::size_t n) - { - reserve(n); - std::memset(m_pos, c, n); - m_pos += n; - } - /// the write position and the end of the writable space, for a writer /// that keeps the position in a local variable (set_cursor() hands it back) char* cursor() const noexcept @@ -6209,9 +6523,27 @@ class output_buffer /// image_check::bounds can have others, which are then written unchanged). inline std::size_t plain_output_run(const unsigned char* s, std::size_t n) noexcept { + std::size_t i = 0; +#if NLOHMANN_VIEW_SSE2 + // (16 bytes per step; with NEON, the mask of a step costs more than it saves on the + // short runs that are typical here) + // NOLINTBEGIN(portability-simd-intrinsics) + for (; i + 16 <= n; i += 16) + { + const __m128i in = _mm_loadu_si128(static_cast(static_cast(s + i))); // NOLINT(bugprone-casting-through-void) + // (a byte is below 0x20 if it is its own minimum with 0x1F) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmpeq_epi8(_mm_min_epu8(in, _mm_set1_epi8(0x1F)), in)); + const auto mask = static_cast(static_cast(_mm_movemask_epi8(special))); + if (mask != 0) + { + return i + static_cast(count_trailing_zeros(mask)); + } + } + // NOLINTEND(portability-simd-intrinsics) +#endif constexpr std::uint64_t ones = 0x0101010101010101ull; constexpr std::uint64_t high = 0x8080808080808080ull; - std::size_t i = 0; for (; i + 8 <= n; i += 8) { const std::uint64_t v = read_eight_bytes(s + i); @@ -6262,6 +6594,70 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept return true; } +#if NLOHMANN_VIEW_VECTOR +/// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a +/// backslash, a control character, DEL, or not ASCII; a signed compare with +/// 0x20 finds control characters and non-ASCII bytes at once), one bit per +/// byte (x86) or one nibble per byte (NEON) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t in = vld1q_u8(p); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)))); + return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20)))); + return static_cast(static_cast(_mm_movemask_epi8(special))); +#endif +} + +/// the width in bits of a byte in the mask of ascii_stop_mask() +constexpr unsigned ascii_stop_bits() noexcept +{ + return NLOHMANN_VIEW_NEON ? 4u : 1u; +} +#endif + +/// As find_ascii_copyable_run(), 16 bytes per step. end is the end of the +/// buffer that holds the n bytes at p (a short run is read as 16 bytes if +/// they are in the buffer). +inline std::size_t ascii_copyable_run(const unsigned char* p, std::size_t n, const unsigned char* end) noexcept +{ +#if NLOHMANN_VIEW_VECTOR + if (n >= 16) + { + std::size_t i = 0; + for (; i + 16 <= n; i += 16) + { + const std::uint64_t mask = ascii_stop_mask(p + i); + if (mask != 0) + { + return i + (static_cast(count_trailing_zeros(mask)) / ascii_stop_bits()); + } + } + if (i == n) + { + return n; + } + // the last 16 bytes: those before i are known to be copyable + const std::uint64_t mask = ascii_stop_mask(p + n - 16); + return mask != 0 ? n - 16 + (static_cast(count_trailing_zeros(mask)) / ascii_stop_bits()) : n; + } + if (end - p >= 16) + { + // (the bytes after the run are masked out) + const std::uint64_t mask = ascii_stop_mask(p) & ((std::uint64_t{1} << (n * ascii_stop_bits())) - 1u); + return mask != 0 ? static_cast(count_trailing_zeros(mask)) / ascii_stop_bits() : n; + } +#else + static_cast(end); +#endif + return find_ascii_copyable_run(p, n); +} + /// A stack that starts in a buffer of the caller (a local array) and moves to /// the heap (a vector of the caller) only when that is full, so that dumps of /// shallow documents need no allocation. The top is a pointer, as in @@ -6341,6 +6737,39 @@ NLOHMANN_VIEW_ALWAYS_INLINE char* put_literal(char* w, const char (&s)[N]) noexc return w + N - 1; } +/// the depth of the innermost open container, counted only for indented output +template +struct depth_counter +{ + std::size_t value = 0; + void enter() noexcept + { + ++value; + } + std::size_t leave() noexcept + { + return --value; + } + std::size_t get() const noexcept + { + return value; + } +}; + +template<> +struct depth_counter +{ + void enter() noexcept {} + std::size_t leave() noexcept // NOLINT(readability-convert-member-functions-to-static): same interface as depth_counter + { + return 0; + } + std::size_t get() const noexcept // NOLINT(readability-convert-member-functions-to-static): same interface as depth_counter + { + return 0; + } +}; + /// how the view's dump() writes a value struct dump_style { @@ -6377,139 +6806,95 @@ class view_serializer void dump(const node* root) { - if (!m_style.pretty) + if (m_style.ensure_ascii) { - if (m_style.ensure_ascii) - { - dump_compact_ascii(root); - } - else if (m_style.source_numbers) - { - dump_compact(root); - } - else - { - dump_compact(root); - } - return; + dump_ascii(root); } - struct frame + else if (m_style.pretty) { - const node* pos; ///< next element, or key of the next member - const node* end; - bool object; - bool first; ///< nothing written yet - }; - std::array buffer; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read - std::vector heap; - small_stack stack(buffer.data(), buffer.size(), heap); - const node* n = root; - for (;;) + dump_pretty(root); + } + else if (m_style.source_numbers) { - // write the value at n - if (is_container(*n)) - { - const bool object = n->kind == static_cast(value_t::object); - if (n->len == 0) - { - m_out.put(object ? "{}" : "[]", 2); - } - else - { - m_out.put(object ? '{' : '['); - stack.push_back(frame{nav::first(m_doc, n), nav::end(m_doc, n), object, true}); - } - } - else - { - write_scalar(*n); - } - - // go to the next value: close finished containers, then separate - for (;;) - { - if (stack.empty()) - { - m_out.finish(); - return; - } - if (stack.back().pos == stack.back().end) - { - const bool object = stack.back().object; - stack.pop_back(); - newline(stack.size()); - m_out.put(object ? '}' : ']'); - continue; - } - frame& f = stack.back(); - if (!f.first) - { - m_out.put(','); - } - f.first = false; - newline(stack.size()); - if (f.object) - { - write_string(*f.pos); - if (m_style.pretty) - { - m_out.put(": "); - } - else - { - m_out.put(':'); - } - n = nav::value(f.pos + 1); - f.pos = document_data::after(f.pos + 1); - } - else - { - n = nav::value(f.pos); - f.pos = document_data::after(f.pos); - } - break; - } + dump_text(root); + } + else + { + dump_text(root); } } private: + /// dump_text() with ensure_ascii, out of line: the walk without it is + /// faster when it is not merged with this one into dump() + NLOHMANN_VIEW_NOINLINE void dump_ascii(const node* root) + { + if (m_style.pretty) + { + if (m_style.source_numbers) + { + dump_text(root); + } + else + { + dump_text(root); + } + } + else if (m_style.source_numbers) + { + dump_text(root); + } + else + { + dump_text(root); + } + } + + /// dump_text() with indentation, out of line: the compact walk is faster + /// when it is not merged with the indented ones into dump() + NLOHMANN_VIEW_NOINLINE void dump_pretty(const node* root) + { + if (m_style.source_numbers) + { + dump_text(root); + } + else + { + dump_text(root); + } + } + /// a string that is not a plain string of the source (decoded, or written /// by an edit), without ensure_ascii: runs without characters to escape - /// are copied - NLOHMANN_VIEW_NOINLINE void write_decoded(const node& n) + /// are copied. If validate is set, the string must be valid UTF-8: the + /// runs are found by the lexer's scan, which validates the sequences, and + /// invalid UTF-8 throws what basic_json::dump() throws. (The compact output + /// writes the bytes of a damaged image unchanged, and scans without + /// validating them.) + NLOHMANN_VIEW_NOINLINE void write_decoded(const node& n, bool validate) { const auto* const s = reinterpret_cast(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) m_out.put('"'); for (std::size_t i = 0; i < n.len;) { - const std::size_t run = plain_output_run(s + i, n.len - i); + const std::size_t run = validate ? string_bulk_run(s + i, n.len - i) : plain_output_run(s + i, n.len - i); if (run != 0) { m_out.put(reinterpret_cast(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) i += run; continue; } + if (validate && s[i] >= 0x80) + { + invalid_utf8(s, n.len); // (the scan stopped at ill-formed UTF-8) + } write_codepoint(s[i], s + i, 1); // a quote, a backslash, or a control character ++i; } m_out.put('"'); } - /// dump_compact() with ensure_ascii, out of line: the walk without it is - /// faster when it is not merged with this one into dump() - NLOHMANN_VIEW_NOINLINE void dump_compact_ascii(const node* root) - { - if (m_style.source_numbers) - { - dump_compact(root); - } - else - { - dump_compact(root); - } - } - - /// whether dump_compact() copies a string or key from the source: one of + /// whether dump_text() copies a string or key from the source: one of /// the source, and with ensure_ascii one without bytes >= 0x7F (an overload /// each, so that the walk without ensure_ascii is compiled exactly as /// before: with GCC, a condition that only folds away later changed its @@ -6524,13 +6909,155 @@ class view_serializer return (n.flags & node_flags::storage) == 0 && is_ascii_without_delete(src + n.off, n.len); } - /// a string with ensure_ascii (dump_compact()), out of line - NLOHMANN_VIEW_NOINLINE void write_ascii_string(const node& n) + /*! + @brief a string with ensure_ascii: the characters that are not printable ASCII as \\uXXXX + (a character beyond U+FFFF as a surrogate pair); strings of a document + are valid UTF-8, except in a damaged image loaded with image_check::bounds, + for which this throws what basic_json::dump() throws for the string + */ + NLOHMANN_VIEW_NOINLINE void write_ascii(const node& n) { - write_string(n); + const auto* const s = reinterpret_cast(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const std::size_t size = n.len; + char* w = m_out.cursor(); + char* lim = m_out.limit(); + // room for k bytes + const auto room = [&](std::size_t k) + { + const auto left = static_cast(lim - w); + if (NLOHMANN_VIEW_UNLIKELY(left < k)) + { + m_out.set_cursor(w); + m_out.reserve(k); + w = m_out.cursor(); + lim = m_out.limit(); + } + }; + static constexpr const char* hex = "0123456789abcdef"; + // \\uXXXX + const auto u_escape = [&](std::uint32_t u) + { + w[0] = '\\'; + w[1] = 'u'; + w[2] = hex[(u >> 12u) & 0xFu]; + w[3] = hex[(u >> 8u) & 0xFu]; + w[4] = hex[(u >> 4u) & 0xFu]; + w[5] = hex[u & 0xFu]; + w += 6; + }; + room(1); + *w++ = '"'; + std::size_t i = 0; + // (a string of the source starts with a run that has nothing to escape, 16 bytes per step) + const std::size_t prefix = (n.flags & node_flags::storage) == 0 ? ascii_copyable_run(s, size, reinterpret_cast(m_doc.src + m_doc.size)) : 0; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + if (prefix != 0) + { + room(prefix); + std::memcpy(w, s, prefix); + w += prefix; + i = prefix; + } + while (i < size) + { + // (a character that needs an escape sequence is not worth a call) + const std::size_t run = is_ascii_copyable(s[i]) ? find_ascii_copyable_run(s + i, size - i) : 0; + if (run != 0) + { + room(run); + std::memcpy(w, s + i, run); + w += run; + i += run; + continue; + } + room(12); + if (s[i] >= 0x80) + { + const unsigned char lead = s[i]; + // (the two-byte sequences of Latin text and the three-byte sequences of most other + // text, with continuation bytes of any value: the ranges that are restricted (E0, + // ED, and the sequences of four bytes) go through validate_one_utf8()) + if (lead >= 0xE1 && lead <= 0xEF && lead != 0xED && size - i >= 3 && (s[i + 1] & 0xC0u) == 0x80 && (s[i + 2] & 0xC0u) == 0x80) + { + u_escape(((lead & 0x0Fu) << 12u) | ((s[i + 1] & 0x3Fu) << 6u) | (s[i + 2] & 0x3Fu)); + i += 3; + continue; + } + if (lead >= 0xC2 && lead <= 0xDF && size - i >= 2 && (s[i + 1] & 0xC0u) == 0x80) + { + u_escape(((lead & 0x1Fu) << 6u) | (s[i + 1] & 0x3Fu)); + i += 2; + continue; + } + const std::size_t len = validate_one_utf8(s + i, size - i); + if (NLOHMANN_VIEW_UNLIKELY(len == 0)) + { + m_out.set_cursor(w); + invalid_utf8(s, size); + return; // LCOV_EXCL_LINE (invalid_utf8 throws) + } + std::uint32_t codepoint = s[i] & (0xFFu >> (len + 1)); + for (std::size_t k = 1; k < len; ++k) + { + codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu); + } + i += len; + if (codepoint <= 0xFFFF) + { + u_escape(codepoint); + } + else + { + u_escape(0xD7C0u + (codepoint >> 10u)); + u_escape(0xDC00u + (codepoint & 0x3FFu)); + } + continue; + } + // a quote, a backslash, or a control character (or DEL) + const unsigned char c = s[i++]; + char escape = 0; + switch (c) + { + case 0x08: + escape = 'b'; + break; + case 0x09: + escape = 't'; + break; + case 0x0A: + escape = 'n'; + break; + case 0x0C: + escape = 'f'; + break; + case 0x0D: + escape = 'r'; + break; + case 0x22: + escape = '"'; + break; + case 0x5C: + escape = '\\'; + break; + default: + break; + } + if (escape != 0) + { + w[0] = '\\'; + w[1] = escape; + w += 2; + } + else + { + u_escape(c); + } + } + room(1); + *w++ = '"'; + m_out.set_cursor(w); } - /// the copies of dump_compact() that are not fixed-size moves (long + /// the copies of dump_text() that are not fixed-size moves (long /// strings, or near the end of the source); out of line, so that the /// compiler does not merge the fixed-size moves into this call NLOHMANN_VIEW_NOINLINE static void copy_long(char* to, const char* from, std::size_t n) noexcept @@ -6547,7 +7074,7 @@ class view_serializer } /*! - @brief the compact output (the default dump(), or with ensure_ascii) + @brief the output, compact (the default dump()) or indented, with or without ensure_ascii The same walk as dump(), with the write position in a local variable (stores through char pointers would otherwise force a reload of the @@ -6555,11 +7082,11 @@ class view_serializer the source copied by fixed-size moves of 32 bytes where the source has that many bytes left, instead of a library call per token. The buffer keeps 64 bytes of slack for the overshoot. With ensure_ascii (Ascii), a - string of the source is copied only if it has no byte >= 0x7F (see - is_ascii_without_delete()); any other string is written by write_string(). + string of the source is copied only if it has nothing to escape (see + copied()); any other string is written by write_ascii(). */ - template - void dump_compact(const node* root) + template + void dump_text(const node* root) { struct frame { @@ -6574,6 +7101,16 @@ class view_serializer const char* const src_end = src + m_doc.size; char* w = m_out.cursor(); char* lim = m_out.limit(); + // (indented output) a line break and 63 indentation characters, from + // which the line break of a level of up to 63 characters is a copy + std::array ruler; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): filled below if used + if (Pretty) + { + ruler.fill(m_style.indent_char); + ruler[0] = '\n'; + } + const std::size_t indent = m_style.indent; + depth_counter level; // depth of the innermost open container (indented output) // room for n bytes and the slack const auto room = [&](std::size_t n) { @@ -6605,6 +7142,22 @@ class view_serializer } w += n; }; + // (indented output) the line break and the indentation of a level + const auto newline = [&](std::size_t depth) + { + const std::size_t count = 1 + (depth * indent); + room(count); + if (count <= ruler.size()) + { + std::memcpy(w, ruler.data(), ruler.size()); + } + else + { + *w = '\n'; + std::memset(w + 1, m_style.indent_char, count - 1); + } + w += count; + }; // a string that is not a plain string of the source (out of line, so // that the cursor stays in a register here) const auto escaped = [&](const node & n) @@ -6612,11 +7165,11 @@ class view_serializer m_out.set_cursor(w); if (enabled(Ascii)) { - write_ascii_string(n); + write_ascii(n); } else { - write_decoded(n); + write_decoded(n, Pretty); } w = m_out.cursor(); lim = m_out.limit(); @@ -6717,6 +7270,7 @@ class view_serializer { *w++ = object ? '{' : '['; stack.push_back(cur); + level.enter(); if (Editable) { cur = frame{nav::first(m_doc, n), nav::end(m_doc, n), object}; @@ -6759,6 +7313,10 @@ class view_serializer { break; } + if (Pretty) + { + newline(level.leave()); + } room(1); *w++ = cur.object ? '}' : ']'; cur = stack.back(); @@ -6767,13 +7325,17 @@ class view_serializer room(1); *w++ = ','; } + if (Pretty) + { + newline(level.get()); + } const node* const at = Editable ? cur.pos : n; if (cur.object) { const node& key = *at; if (copied(key, src, std::integral_constant())) { - room(key.len + 3); + room(key.len + (Pretty ? 4 : 3)); *w++ = '"'; copy(src + key.off, key.len); w[0] = '"'; @@ -6783,9 +7345,13 @@ class view_serializer else { escaped(key); - room(1); + room(Pretty ? 2 : 1); *w++ = ':'; } + if (Pretty) + { + *w++ = ' '; + } if (Editable) { n = nav::value(at + 1); @@ -6804,69 +7370,6 @@ class view_serializer } } - void newline(std::size_t level) - { - if (m_style.pretty) - { - m_out.put('\n'); - m_out.put_repeated(m_style.indent_char, level * m_style.indent); - } - } - - void write_scalar(const node& n) - { - switch (static_cast(n.kind)) - { - case value_t::null: - m_out.put("null"); - break; - case value_t::boolean: - if ((n.flags & node_flags::is_true) != 0) - { - m_out.put("true"); - } - else - { - m_out.put("false"); - } - break; - case value_t::string: - write_string(n); - break; - case value_t::number_integer: - case value_t::number_unsigned: - { - const char* const token = m_doc.str(n); - const std::uint32_t len = number_length(n); - if (!m_style.source_numbers && len == 2 && token[0] == '-' && token[1] == '0') - { - m_out.put('0'); // parse() reads -0 as the integer 0 - } - else - { - m_out.put(token, len); - } - break; - } - case value_t::number_float: - if (m_style.source_numbers && (n.flags & node_flags::storage) != node_flags::edited) - { - m_out.put(m_doc.str(n), n.len); // (a float set by an edit is written as with shortest) - } - else - { - write_float_node(n); - } - break; - case value_t::object: // LCOV_EXCL_LINE (containers are written by dump()) - case value_t::array: // LCOV_EXCL_LINE - case value_t::binary: // LCOV_EXCL_LINE (not in a document) - case value_t::discarded: // LCOV_EXCL_LINE - default: // LCOV_EXCL_LINE - break; // LCOV_EXCL_LINE - } - } - /// a float node as dump() writes it void write_float_node(const node& n) { @@ -6977,72 +7480,6 @@ class view_serializer m_out.put(s.data(), s.size()); } - void write_string(const node& n) - { - const char* const s = m_doc.str(n); - m_out.put('"'); - if ((n.flags & node_flags::storage) == 0 && (!m_style.ensure_ascii || is_ascii_without_delete(s, n.len))) - { - // a string of the source without escape sequences has nothing to - // escape (with ensure_ascii, if it is printable ASCII) - m_out.put(s, n.len); - } - else if (m_style.ensure_ascii) - { - write_escaped(reinterpret_cast(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - else - { - write_escaped(reinterpret_cast(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - m_out.put('"'); - } - - /// as serializer::dump_escaped(); strings of a document are valid UTF-8, - /// except in a damaged image loaded with image_check::bounds, for which - /// this throws what basic_json::dump() throws for the string - template - void write_escaped(const unsigned char* s, std::size_t n) - { - std::size_t i = 0; - while (i < n) - { - std::size_t run = 0; - if (!EnsureAscii) - { - run = string_bulk_run(s + i, n - i); - } - else if (is_ascii_copyable(s[i])) - { - run = find_ascii_copyable_run(s + i, n - i); - } - if (run != 0) - { - m_out.put(reinterpret_cast(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - i += run; - continue; - } - std::uint32_t codepoint = s[i]; - std::size_t len = 1; - if (codepoint >= 0x80) - { - len = validate_one_utf8(s + i, n - i); - if (NLOHMANN_VIEW_UNLIKELY(len == 0)) - { - invalid_utf8(s, n); - return; - } - codepoint &= 0xFFu >> (len + 1); - for (std::size_t k = 1; k < len; ++k) - { - codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu); - } - } - write_codepoint(codepoint, s + i, len); - i += len; - } - } - /// throw what basic_json::dump() throws for a string that is not valid UTF-8 NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n) { @@ -7590,7 +8027,7 @@ class basic_json_view } detail::view::throw_type_error(305, "cannot use operator[] with a numeric argument with ", type_name()); } - return idx < m_node->len ? basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))) : basic_json_view(); + return idx < m_node->len ? element(idx) : basic_json_view(); } /// any other integer type (int, unsigned, long, std::int64_t, ...; a @@ -7653,7 +8090,7 @@ class basic_json_view { detail::view::throw_out_of_range(401, detail::concat("array index ", std::to_string(idx), " is out of range")); } - return basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))); + return element(idx); } /// any other integer type, see operator[]; negative values are out of @@ -7687,7 +8124,13 @@ class basic_json_view string_t value(string_view_t key, const char* default_value) const { - return value(key, string_t(default_value)); + if (NLOHMANN_VIEW_UNLIKELY(!is_object())) + { + detail::view::throw_type_error(306, "cannot use value() with ", type_name()); + } + // (the default is made only if there is no such member) + const basic_json_view r = lookup(key); + return r.is_discarded() ? string_t(default_value) : r.template get(); } /// the value a JSON pointer refers to converted to T, or the default @@ -7706,7 +8149,12 @@ class basic_json_view string_t value(const json_pointer& ptr, const char* default_value) const { - return value(ptr, string_t(default_value)); + if (NLOHMANN_VIEW_UNLIKELY(!is_structured())) + { + detail::view::throw_type_error(306, "cannot use value() with ", type_name()); + } + const basic_json_view r = detail::view::resolve_pointer(*this, detail::json_pointer_access::reference_tokens(ptr), detail::view::pointer_mode::value); + return r.is_discarded() ? string_t(default_value) : r.template get(); } /// the first element or member value; a primitive value itself. Throws @@ -7728,7 +8176,8 @@ class basic_json_view { if (is_structured() && m_node->len != 0) { - return basic_json_view(m_doc, navigation::value(detail::view::last_child(*m_doc, m_node) + (is_object() ? 1 : 0))); + return unedited() ? basic_json_view(m_doc, detail::view::last_child(*m_doc, m_node) + (is_object() ? 1 : 0)) + : basic_json_view(m_doc, navigation::value(detail::view::last_child(*m_doc, m_node) + (is_object() ? 1 : 0))); } return front(); } @@ -7745,7 +8194,7 @@ class basic_json_view { return end(); } - const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + const node* const k = find_key(key); return k != nullptr ? iterator(m_doc, k, true) : end(); } @@ -7762,7 +8211,7 @@ class basic_json_view /// whether this is an object with a member with this key bool contains(string_view_t key) const { - return is_object() && detail::view::find_member(*m_doc, m_node, key.data(), key.size()) != nullptr; + return is_object() && find_key(key) != nullptr; } bool contains(const char* key) const @@ -7922,7 +8371,14 @@ class basic_json_view // the compact writer keeps 64 bytes of slack, so that it does not grow // the buffer just before the end const std::size_t estimate = source_extent() + (style.pretty ? source_extent() / 2 : 0) + 160; - detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); + if (unedited()) + { + detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); + } + else + { + detail::view::view_serializer(*m_doc, out, estimate, style).dump(m_node); + } return out; } @@ -8002,7 +8458,7 @@ class basic_json_view { return BasicJsonType(value_t::discarded); } - return detail::view::materialize(*m_doc, m_node); + return unedited() ? detail::view::materialize(*m_doc, m_node) : detail::view::materialize(*m_doc, m_node); } /// byte offset of this value in the source text (for strings: of the @@ -8080,10 +8536,50 @@ class basic_json_view /// (object required) NLOHMANN_VIEW_ALWAYS_INLINE basic_json_view lookup(string_view_t key) const noexcept { + if (unedited()) + { + const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + return k != nullptr ? basic_json_view(m_doc, k + 1) : basic_json_view(); + } const node* const k = detail::view::find_member(*m_doc, m_node, key.data(), key.size()); return k != nullptr ? basic_json_view(m_doc, navigation::value(k + 1)) : basic_json_view(); } + /// whether the document has not been edited yet: until the first edit, its index is walked like that of a + /// read-only document (nothing is moved or linked), which is cheaper + NLOHMANN_VIEW_ALWAYS_INLINE bool unedited() const noexcept + { + return !navigation::editable() || m_doc->edits == nullptr; + } + + /// the member of this object with @a key (the last one if the key occurs more than once), or nullptr + NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key) const + { + return find_key(key, std::integral_constant {}); + } + + NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key, std::true_type /*editable*/) const + { + // (an editable document without edits is looked up as one that cannot be edited) + return unedited() ? detail::view::find_member(*m_doc, m_node, key.data(), key.size()) + : detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + } + + NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key, std::false_type /*editable*/) const + { + return detail::view::find_member(*m_doc, m_node, key.data(), key.size()); + } + + /// the element of an array at an index below its size + NLOHMANN_VIEW_ALWAYS_INLINE basic_json_view element(size_type idx) const + { + if (unedited()) + { + return basic_json_view(m_doc, detail::view::element_at(*m_doc, m_node, idx)); + } + return basic_json_view(m_doc, navigation::value(detail::view::element_at(*m_doc, m_node, idx))); + } + // --- get() dispatch --- bool get_impl(detail::view::value_tag /*unused*/, detail::priority_tag<2> /*unused*/) const @@ -8823,7 +9319,8 @@ class basic_json_document // NOLINTBEGIN(readability-avoid-nested-conditional-operator): a constant expression of C++11 append_input_impl(buf, std::forward(input), std::integral_constant < int, detail::is_contiguous_byte_container::value ? 0 : std::is_base_of::value ? 1 - : detail::view::is_adapter_input::value ? 2 : 3 > {}); + : detail::view::is_adapter_input::value ? 2 + : detail::view::is_c_stream::value ? 4 : 3 > {}); // NOLINTEND(readability-avoid-nested-conditional-operator) } @@ -8845,6 +9342,19 @@ class basic_json_document detail::view::append_adapter(buf, std::move(input.ia)); } +#ifndef JSON_NO_IO + static void append_input_impl(std::string& buf, std::FILE* file, std::integral_constant /*C stream*/) + { + if (file == nullptr) + { + // (as before: the library's adapter decides what a null stream yields) + detail::view::append_adapter(buf, detail::input_adapter(file)); + return; + } + detail::view::append_file(buf, file); + } +#endif + template static void append_input_impl(std::string& buf, T&& input, std::integral_constant /*other*/) { diff --git a/src/modules/json.cppm b/src/modules/json.cppm index ffc9e00e2..3917e0722 100644 --- a/src/modules/json.cppm +++ b/src/modules/json.cppm @@ -60,6 +60,7 @@ inline namespace json_literals namespace detail { using NLOHMANN_JSON_NAMESPACE::detail::json_sax_dom_callback_parser; + using NLOHMANN_JSON_NAMESPACE::detail::json_sax_dom_parser; using NLOHMANN_JSON_NAMESPACE::detail::unknown_size; } // namespace detail diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp index b894d3f7f..1e18c3f15 100644 --- a/tests/src/unit-binary_writer_sinks.cpp +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -492,3 +492,278 @@ TEST_CASE("binary_reserve_hint never over-reserves") CHECK(nlohmann::detail::binary_reserve_hint(json({{"a", 1}, {"b", 2}})) == 5); } } + +namespace +{ + +// numbers around every boundary of the integer encodings, and floats of each +// kind (exact in float, needing double, special values) +std::vector boundary_numbers() +{ + std::vector numbers; + for (int k = 0; k < 64; ++k) + { + const std::uint64_t p = std::uint64_t{1} << k; + for (const std::uint64_t d : + { + p - 1, p, p + 1 + }) + { + numbers.emplace_back(d); + if (d <= static_cast((std::numeric_limits::max)())) + { + numbers.emplace_back(static_cast(d)); + numbers.emplace_back(-static_cast(d)); + numbers.emplace_back(-static_cast(d) - 1); + } + } + } + numbers.emplace_back((std::numeric_limits::min)()); + numbers.emplace_back((std::numeric_limits::max)()); + for (const double f : + { + 0.0, -0.0, 1.0, -1.0, 0.5, 0.1, 3.25, 1e300, -1e-300, 3.4028234663852886e38, 1e39, + std::numeric_limits::infinity(), -std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN() + }) + { + numbers.emplace_back(f); + } + return numbers; +} + +// the concatenation of the encodings of the elements of @a numbers[first, first + count) +template +std::vector concat_singles(const std::vector& numbers, const std::size_t first, const std::size_t count, Single single) +{ + std::vector bytes; + for (std::size_t i = first; i < first + count; ++i) + { + const auto one = single(numbers[i]); + bytes.insert(bytes.end(), one.begin(), one.end()); + } + return bytes; +} + +} // namespace + +// Runs of numbers in arrays are encoded straight into space reserved from the +// sink, by code that is separate from the encoding of a single value. Both have +// to agree on every number: an array is its head and the encodings of its +// elements, one after the other. +TEST_CASE("runs of numbers in arrays are encoded as the numbers one by one") +{ + const std::vector numbers = boundary_numbers(); + + // arrays of every length that a one-byte head covers, so that runs are as + // long as, and longer and shorter than, what fits one reservation + for (const std::size_t count : + { + 1u, 2u, 3u, 5u, 14u, 15u, 23u + }) + { + for (std::size_t first = 0; first + count <= numbers.size(); first += 7) + { + const json array(std::vector(numbers.begin() + static_cast(first), + numbers.begin() + static_cast(first + count))); + CAPTURE(array.dump(-1, ' ', false, json::error_handler_t::replace)) + + std::vector expected{static_cast(0x80 + count)}; + auto elements = concat_singles(numbers, first, count, [](const json & j) + { + return json::to_cbor(j); + }); + expected.insert(expected.end(), elements.begin(), elements.end()); + CHECK(json::to_cbor(array) == expected); + + expected.assign(1, static_cast(count <= 15 ? 0x90 + count : 0xDC)); + if (count > 15) + { + expected.push_back(0); + expected.push_back(static_cast(count)); + } + elements = concat_singles(numbers, first, count, [](const json & j) + { + return json::to_msgpack(j); + }); + expected.insert(expected.end(), elements.begin(), elements.end()); + CHECK(json::to_msgpack(array) == expected); + + for (const bool bjdata : + { + false, true + }) + { + expected.assign(1, static_cast('[')); + elements = concat_singles(numbers, first, count, [bjdata](const json & j) + { + return bjdata ? json::to_bjdata(j) : json::to_ubjson(j); + }); + expected.insert(expected.end(), elements.begin(), elements.end()); + expected.push_back(static_cast(']')); + CHECK((bjdata ? json::to_bjdata(array) : json::to_ubjson(array)) == expected); + } + + // BON8 has no integers above the int64 range + bool representable = true; + for (std::size_t i = first; i < first + count; ++i) + { + representable = representable && bon8_representable(numbers[i]); + } + if (representable) + { + expected.assign(1, static_cast(count <= 4 ? 0x80 + count : 0x85)); + elements = concat_singles(numbers, first, count, [](const json & j) + { + return json::to_bon8(j); + }); + expected.insert(expected.end(), elements.begin(), elements.end()); + if (count > 4) + { + expected.push_back(0xFE); + } + CHECK(json::to_bon8(array) == expected); + } + else + { + CHECK_THROWS_AS(json::to_bon8(array), json::out_of_range&); + } + } + } +} + +// An array that is written in more than one reservation (more numbers than fit +// one), and arrays of an index with more digits, round trip for every format. +TEST_CASE("large arrays and many digits of indices round trip") +{ + json array = json::array(); + for (int i = 0; i < 12000; ++i) + { + switch (i % 4) + { + case 0: + array.push_back(i); + break; + case 1: + array.push_back(-i); + break; + case 2: + array.push_back(i * 0.25); + break; + default: + array.push_back(i % 8 == 3); + break; + } + } + json document = json::object(); + document["numbers"] = array; + document["nested"] = json::array({array, "text", array}); + + CHECK(json::from_cbor(json::to_cbor(document)) == document); + CHECK(json::from_msgpack(json::to_msgpack(document)) == document); + CHECK(json::from_ubjson(json::to_ubjson(document)) == document); + CHECK(json::from_ubjson(json::to_ubjson(document, true, true)) == document); + CHECK(json::from_bjdata(json::to_bjdata(document)) == document); + CHECK(json::from_bson(json::to_bson(document)) == document); + CHECK(json::from_bon8(json::to_bon8(document)) == document); +} + +// Strings and keys are written with their head in one reservation up to a +// length; around that length, and with every kind of head, they must read back. +TEST_CASE("strings of every head length round trip") +{ + json document = json::object(); + json array = json::array(); + for (const std::size_t length : + { + 0u, 1u, 2u, 3u, 4u, 7u, 8u, 15u, 16u, 17u, 23u, 24u, 25u, 31u, 32u, 33u, 63u, 127u, 128u, 129u, 200u, 245u, 246u, 247u, 248u, 249u, 250u, 251u, 252u, 253u, 254u, 255u, 256u, 257u, 300u, 1000u, 65535u, 65536u, 70000u + }) + { + const std::string text(length, 'x'); + array.push_back(text); + document[std::string(length, 'k')] = text; + } + document["array"] = array; + + CHECK(json::from_cbor(json::to_cbor(document)) == document); + CHECK(json::from_msgpack(json::to_msgpack(document)) == document); + CHECK(json::from_ubjson(json::to_ubjson(document)) == document); + CHECK(json::from_ubjson(json::to_ubjson(document, true, true)) == document); + CHECK(json::from_bjdata(json::to_bjdata(document)) == document); + CHECK(json::from_bson(json::to_bson(document)) == document); + CHECK(json::from_bon8(json::to_bon8(document)) == document); +} + +#if !defined(JSON_NOEXCEPTION) +// What a write that throws leaves behind in an output adapter is what was written +// before it, whatever the adapter and however the adapter sink buffers. (Without +// exceptions, the writes in CHECK_THROWS_AS do not run.) +TEST_CASE("a write that throws leaves the same bytes in every kind of output") +{ + json array = json::array(); + for (int i = 0; i < 700; ++i) + { + array.push_back(i * 1000); + array.push_back("text " + std::to_string(i)); + } + array.push_back("bad \xFF byte"); + json document = json::object(); + document["first"] = array; + document["second"] = 1; + + const auto strict = json::error_handler_t::strict; + + SECTION("cbor") + { + std::vector as_vector; + std::string as_string; + std::ostringstream as_stream; + CHECK_THROWS_AS(json::to_cbor(document, as_vector, strict), json::type_error&); + CHECK_THROWS_AS(json::to_cbor(document, as_string, strict), json::type_error&); + CHECK_THROWS_AS(json::to_cbor(document, as_stream, strict), json::type_error&); + CHECK(!as_vector.empty()); + CHECK(std::string(reinterpret_cast(as_vector.data()), as_vector.size()) == as_string); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + CHECK(as_stream.str() == as_string); + } + + SECTION("msgpack") + { + std::vector as_vector; + std::string as_string; + std::ostringstream as_stream; + CHECK_THROWS_AS(json::to_msgpack(document, as_vector, strict), json::type_error&); + CHECK_THROWS_AS(json::to_msgpack(document, as_string, strict), json::type_error&); + CHECK_THROWS_AS(json::to_msgpack(document, as_stream, strict), json::type_error&); + CHECK(!as_vector.empty()); + CHECK(std::string(reinterpret_cast(as_vector.data()), as_vector.size()) == as_string); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + CHECK(as_stream.str() == as_string); + } + + SECTION("ubjson") + { + std::vector as_vector; + std::string as_string; + std::ostringstream as_stream; + CHECK_THROWS_AS(json::to_ubjson(document, as_vector, false, false, strict), json::type_error&); + CHECK_THROWS_AS(json::to_ubjson(document, as_string, false, false, strict), json::type_error&); + CHECK_THROWS_AS(json::to_ubjson(document, as_stream, false, false, strict), json::type_error&); + CHECK(!as_vector.empty()); + CHECK(std::string(reinterpret_cast(as_vector.data()), as_vector.size()) == as_string); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + CHECK(as_stream.str() == as_string); + } + + SECTION("bson") + { + // the size pass finds the string before anything is written + std::vector as_vector; + std::string as_string; + std::ostringstream as_stream; + CHECK_THROWS_AS(json::to_bson(document, as_vector, strict), json::type_error&); + CHECK_THROWS_AS(json::to_bson(document, as_string, strict), json::type_error&); + CHECK_THROWS_AS(json::to_bson(document, as_stream, strict), json::type_error&); + CHECK(as_vector.empty()); + CHECK(as_string.empty()); + CHECK(as_stream.str().empty()); + } +} +#endif diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 1d1b5f40b..dd78f7ff4 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2952,3 +2952,81 @@ TEST_CASE("diagnostic positions: value lifetime, input adapters, and SAX") } } #endif + +TEST_CASE("accept() agrees with parse() on floats around the overflow limit") +{ + // accept() does not convert a float that cannot overflow; the tokens + // around the limit (and beyond it) must still be rejected exactly as + // parse() rejects them + const std::vector numbers = + { + "1e0", "123.456e-5", "1e-400", "-1e-400", "1e307", "1e308", "1e309", "-1e309", + "10e307", "100e307", "0.1e309", "0.1e310", "1E+308", "1E+309", + "1.7976931348623157e308", "1.7976931348623159e308", "1.8e308", "-1.7976931348623157e308", + "12345678901234567e291", "12345678901234567e292", "123456789012345678e290", "123456789012345678e291", + "1234567890123456789e289", "1234567890123456789e290", "9999999999999999999e289", "9999999999999999999e290", + "0.00000000000000000001e328", "0.00000000000000000001e329" + }; + + for (const char* number : numbers) + { + for (const std::string& text : + { + std::string(number), "[" + std::string(number) + "]", "{\"a\":" + std::string(number) + "}" + }) + { + INFO(text); + const bool parses = !json::parse(text, nullptr, false).is_discarded(); + CHECK(json::accept(text) == parses); + } + } +} + +namespace +{ +// a DOM parser that reports parse errors without throwing, as in the sax_no_exception example +template +class quiet_dom_parser : public nlohmann::detail::json_sax_dom_parser +{ + public: + explicit quiet_dom_parser(BasicJsonType& j) + : nlohmann::detail::json_sax_dom_parser(j, false) + {} + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const nlohmann::detail::exception& /*unused*/) + { + return false; + } +}; + +template +std::string read_before_error(const std::string& text) +{ + BasicJsonType j; + quiet_dom_parser sax(j); + CHECK(!BasicJsonType::sax_parse(text, &sax)); + return j.dump(); +} +} // namespace + +TEST_CASE("a json_sax_dom_parser keeps the values read before a parse error") +{ + // a json_sax_dom_parser that a user constructs stores every value in place, so a + // handler that does not throw on a parse error sees what was read before it + const std::vector> cases = + { + {"[1,2,3,]", "[1,2,3]"}, + {"[[1,[2,3],4],[5,", "[[1,[2,3],4],[5]]"}, + {R"({"a":1,"b":[1,2,{"c":)", R"({"a":1,"b":[1,2,{"c":null}]})"}, + {R"({"a":1,"a":)", R"({"a":1})"}, + {"[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,]", "[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]"}, + {R"({"k0":0,"k1":1,"k2":{"x":[true,)", R"({"k0":0,"k1":1,"k2":{"x":[true]}})"}, + {R"([{},[],{"a":[]},x)", R"([{},[],{"a":[]}])"}, + }; + for (const auto& c : cases) + { + CAPTURE(c.first) + CHECK(read_before_error(c.first) == c.second); + CHECK(read_before_error(c.first) == c.second); + } +} diff --git a/tests/src/unit-custom-number-types.cpp b/tests/src/unit-custom-number-types.cpp index f9c197574..e324417db 100644 --- a/tests/src/unit-custom-number-types.cpp +++ b/tests/src/unit-custom-number-types.cpp @@ -285,6 +285,58 @@ TEST_CASE("custom number types: integers beyond 64 bits") CHECK(wide_json::to_bjdata(wide_json::array({uint_big, uint_big}), true, true) == expected_bjdata_array); } + SECTION("integers in arrays and objects") + { + // runs of numbers and scalar object members are encoded by loops of + // their own for the number types up to 64 bits; a wider number type + // takes the generic path, which checks the range of every integer + const wide_json run = wide_json::array({make_int(1), make_int(-3), make_uint(7), wide_json(1.5), int_big, make_int(5)}); + const wide_json run_negative = wide_json::array({make_int(1), make_int(-3), int_big_negative}); + const wide_json run_unsigned = wide_json::array({make_uint(1), make_uint(7), uint_big}); + const wide_json members = wide_json::object({{"a", make_int(1)}, {"b", wide_json(1.5)}, {"c", int_big}}); + + const std::vector failing = {run, run_negative, run_unsigned, members, wrap(run), wrap(members)}; + for (const auto& value : failing) + { + CHECK_THROWS_AS(_ = wide_json::to_cbor(value), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_msgpack(value), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(value)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bon8(value), out_of_range&); + } + +#if !defined(JSON_NOEXCEPTION) + // a failing write leaves no more than the elements before the failing one + // (without exceptions, the writes in CHECK_THROWS_AS do not run) + std::vector out; + CHECK_THROWS_AS(wide_json::to_cbor(run, out), out_of_range&); + // (the head of the array, 1, -3, 7, and 1.5 as a single-precision float) + CHECK(out == std::vector({0x86, 0x01, 0x22, 0x07, 0xFA, 0x3F, 0xC0, 0x00, 0x00})); + out.clear(); + CHECK_THROWS_AS(wide_json::to_bon8(run, out), out_of_range&); + CHECK(!out.empty()); +#endif + + // the same values that fit are written exactly as with 64-bit number types + const wide_json fits = wide_json::array({make_int(1), make_int(-3), make_uint(7), wide_json(1.5), make_int(min_int64), make_int(max_int64), make_int(200), make_int(-70000)}); + const nlohmann::json fits64 = nlohmann::json::array({1, -3, 7, 1.5, (std::numeric_limits::min)(), (std::numeric_limits::max)(), 200, -70000}); + CHECK(wide_json::to_cbor(fits) == nlohmann::json::to_cbor(fits64)); + CHECK(wide_json::to_msgpack(fits) == nlohmann::json::to_msgpack(fits64)); + CHECK(wide_json::to_ubjson(fits) == nlohmann::json::to_ubjson(fits64)); + CHECK(wide_json::to_ubjson(fits, true, true) == nlohmann::json::to_ubjson(fits64, true, true)); + CHECK(wide_json::to_bjdata(fits) == nlohmann::json::to_bjdata(fits64)); + CHECK(wide_json::to_bjdata(fits, true, true) == nlohmann::json::to_bjdata(fits64, true, true)); + CHECK(wide_json::to_bon8(fits) == nlohmann::json::to_bon8(fits64)); + CHECK(wide_json::to_bson(wrap(fits)) == nlohmann::json::to_bson(nlohmann::json::object({{"v", fits64}}))); + + const wide_json fit_members = wide_json::object({{"a", make_int(1)}, {"b", make_int(-300)}, {"c", make_uint(max_uint64)}, {"d", wide_json(2.5)}}); + const nlohmann::json fit_members64 = nlohmann::json::object({{"a", 1}, {"b", -300}, {"c", (std::numeric_limits::max)()}, {"d", 2.5}}); + CHECK(wide_json::to_cbor(fit_members) == nlohmann::json::to_cbor(fit_members64)); + CHECK(wide_json::to_msgpack(fit_members) == nlohmann::json::to_msgpack(fit_members64)); + CHECK(wide_json::to_ubjson(fit_members) == nlohmann::json::to_ubjson(fit_members64)); + CHECK(wide_json::to_bjdata(fit_members) == nlohmann::json::to_bjdata(fit_members64)); + CHECK(wide_json::to_bson(fit_members) == nlohmann::json::to_bson(fit_members64)); + } + SECTION("BJData ND-array") { // an ND-array element beyond 64 bits does not fit any dtype, so the diff --git a/tests/src/unit-json_view.cpp b/tests/src/unit-json_view.cpp index caadee629..e87162d98 100644 --- a/tests/src/unit-json_view.cpp +++ b/tests/src/unit-json_view.cpp @@ -185,6 +185,33 @@ TEST_CASE("json_view") // duplicate keys: the last value, at the position of the first key CHECK(materialized(R"({"a":1,"b":2,"a":3})") == json::parse(R"({"a":1,"b":2,"a":3})")); CHECK(materialized(R"({"a":1,"b":2,"a":3})").dump() == R"({"a":3,"b":2})"); + // scalars (made without the handler), and nesting around the depth up to which the stack of open containers is local + for (const char* scalar : + {"null", "true", "false", "0", "-7", "18446744073709551615", "1.5e300", "-0.0", "\"\"", "\"short\"", "\"a string that is longer than fifteen bytes\"", R"("\u00e4\n")" + }) + { + CAPTURE(scalar) + CHECK(materialized(scalar) == json::parse(scalar)); + const std::string in_array = std::string("[1,") + scalar + "]"; + const json_document d = json_document::parse(in_array); + CHECK(d.root()[1].materialize() == json::parse(scalar)); + CHECK(d.root()[1].get() == json::parse(scalar)); + } + for (const std::size_t levels : + { + std::size_t{1}, std::size_t{15}, std::size_t{16}, std::size_t{17}, std::size_t{32}, std::size_t{33} + }) + { + CAPTURE(levels) + const std::string nested = std::string(levels, '[') + R"({"k":[1,{"j":2}]})" + std::string(levels, ']'); + CHECK(materialized(nested) == json::parse(nested)); + std::string wide = R"({"a":)"; + wide += nested; + wide += R"(,"b":)"; + wide += nested; + wide += "}"; + CHECK(materialized(wide) == json::parse(wide)); + } // very deep nesting (iterative, as parse()) const std::string deep = std::string(100000, '[') + std::string(100000, ']'); // (one check per step, so that a failure shows which step it is) @@ -410,6 +437,31 @@ TEST_CASE("json_view") std::istringstream rvalue_source("[1]"); CHECK(json_document::parse_copy(std::move(rvalue_source)).owns_source()); + // a C stream is read in blocks as well (more than one block, and a + // text whose length is a multiple of the block size) + for (const std::size_t extra : + { + std::size_t{0}, std::size_t{1} + }) + { + std::FILE* long_file = std::tmpfile(); // NOLINT(cppcoreguidelines-owning-memory) + if (long_file != nullptr) + { + std::string padded = text; + padded.append(((4096 - (padded.size() % 4096)) % 4096) + extra, ' '); + CHECK(std::fwrite(padded.data(), 1, padded.size(), long_file) == padded.size()); + CHECK(std::fseek(long_file, 0, SEEK_SET) == 0); + const json_document from_long_file = json_document::parse(long_file); + CHECK(from_long_file.source() == padded); + CHECK(from_long_file.root().materialize() == json::parse(text)); + CHECK(std::fclose(long_file) == 0); // NOLINT(cppcoreguidelines-owning-memory) + } + } + { + json_document from_null; + CHECK_THROWS_AS(from_null = json_document::parse(static_cast(nullptr)), json::parse_error&); + } + // a stream without a buffer is an empty input std::istream no_buffer(nullptr); CHECK(json_document::parse(no_buffer, false).is_discarded()); @@ -1006,6 +1058,49 @@ TEST_CASE("json_view element access and iteration") #endif } + SECTION("arrays whose elements have no subtrees: index and back()") + { + // (the elements are adjacent in the index; an array, an object, or an + // empty container anywhere makes the lookup skip element by element) + const std::vector texts = + { + "[]", "[1]", "[1,2,3]", "[1,\"two\",3.5,true,null,-7,18446744073709551615]", "[[]]", "[{}]", "[1,[2],3]", + "[1,2,{\"a\":3}]", "[[1,2],3,4]", "[1,2,3,[4,5,6,7,8],9,10,11,12]", "[{\"a\":[1]},2]", R"(["a\u00e4","b"])" + }; + for (const std::string& text : texts) + { + CAPTURE(text) + const json_document d = json_document::parse(text); + const json_view v = d.root(); + const json expected = json::parse(text); + for (std::size_t i = 0; i < expected.size(); ++i) + { + CHECK(v[i].materialize() == expected[i]); + CHECK(v.at(i).materialize() == expected[i]); + } + CHECK(v[expected.size()].is_discarded()); + if (!expected.empty()) + { + CHECK(v.back().materialize() == expected.back()); + } + } + // a long array (the direct lookup is not limited to short ones), also with a container at its end + std::string text = "[0"; + for (int i = 1; i < 1000; ++i) + { + text += "," + std::to_string(i); + } + const json_document scalars = json_document::parse(text + "]"); + CHECK(scalars.root()[999].get() == 999); + CHECK(scalars.root()[500].get() == 500); + CHECK(scalars.root().at(1).get() == 1); + CHECK(scalars.root().back().get() == 999); + const json_document container_last = json_document::parse(text + ",[1000]]"); + CHECK(container_last.root()[999].get() == 999); + CHECK(container_last.root()[1000][0].get() == 1000); + CHECK(container_last.root().back()[0].get() == 1000); + } + SECTION("iterators") { const json_document d = json_document::parse(R"({"x":[1,{"y":2}],"z":null})"); @@ -1045,6 +1140,28 @@ TEST_CASE("json_view element access and iteration") CHECK((*first++).key() == "0"); CHECK((*first).key() == "1"); CHECK(++first == items.end()); + + // the keys of array elements are their indexes, whether or not they are asked for + std::string numbers = "[0"; + for (int i = 1; i < 1234; ++i) + { + numbers += "," + std::to_string(i); + } + const json_document array = json_document::parse(numbers + "]"); + std::size_t position = 0; + for (const auto& item : array.root().items()) + { + CHECK(item.value().get() == position); + if (position % 3 != 0) // (not for every item: values alone do not make the key) + { + CHECK(item.key() == std::to_string(position)); + CHECK(item.key().data() == item.key().data()); // (the same text every time) + const json_view::item copy = item; + CHECK(copy.key() == std::to_string(position)); + } + ++position; + } + CHECK(position == 1234); #ifdef JSON_HAS_CPP_17 std::string pairs; for (const auto [key, value] : d.root().items()) @@ -1383,7 +1500,17 @@ TEST_CASE("json_view values") CHECK(exception_of([&] { static_cast(v.value("s", 0)); }) == without_path(exception_of([&] { static_cast(j.value("s", 0)); }))); CHECK(exception_of([&] { static_cast(v["n"].value("x", 0)); }) == without_path(exception_of([&] { static_cast(j["n"].value("x", 0)); }))); CHECK(exception_of([&] { static_cast(v["n"].value(json::json_pointer("/x"), 0)); }) == without_path(exception_of([&] { static_cast(j["n"].value(json::json_pointer("/x"), 0)); }))); + // string defaults: the member is not a string, or the value is neither an object nor an array + CHECK(exception_of([&] { static_cast(v.value("n", "default")); }) == without_path(exception_of([&] { static_cast(j.value("n", "default")); }))); + CHECK(exception_of([&] { static_cast(v["n"].value("x", "default")); }) == without_path(exception_of([&] { static_cast(j["n"].value("x", "default")); }))); + CHECK(exception_of([&] { static_cast(v.value(json::json_pointer("/n"), "default")); }) == without_path(exception_of([&] { static_cast(j.value(json::json_pointer("/n"), "default")); }))); + CHECK(exception_of([&] { static_cast(v["s"].value(json::json_pointer("/x"), "default")); }) == without_path(exception_of([&] { static_cast(j["s"].value(json::json_pointer("/x"), "default")); }))); #endif + // (defaults and members longer than the short string optimization) + CHECK(v.value("s", "a default longer than fifteen characters") == "text"); + CHECK(v.value("missing", "a default longer than fifteen characters") == "a default longer than fifteen characters"); + CHECK(v.value(json::json_pointer("/s"), "a default longer than fifteen characters") == "text"); + CHECK(v.value(json::json_pointer("/zz"), "a default longer than fifteen characters") == "a default longer than fifteen characters"); } } diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index b43af739d..6935981fb 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -523,6 +523,58 @@ TEST_CASE("json_view builder: strings across vector blocks") } } +TEST_CASE("json_view builder: escapes between runs of text") +{ + // The text between escapes is copied 16 bytes at a time where there are + // that many bytes left, and with the UTF-8 check where there is + // non-ASCII text. Runs of every length from 0 to 40 between valid and + // invalid escapes, UTF-8 sequences, and control characters, in strings + // that are cut at all places near the end of the text. + const std::array valid = {{"\\n", "\\\"", "\\\\", "\\/", "\\t", "\\u00e4", "\\u20ac", "\\ud83d\\ude00", "\\u0000"}}; + const std::array invalid = {{"\\x", "\\ud800", "\\udc00", "\\u12G4", "\\u12", "\\ud83dx", "\\ud83d\\u0041", "\\", "\x01"}}; + const std::array utf8 = {{"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xc3", "\xff", "\x80"}}; + std::mt19937 rng(2026); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t run = 0; run <= 40; ++run) + { + for (int i = 0; i < 60; ++i) + { + std::string content = std::string(rng() % 20, 'a'); + const auto pieces = 1 + (rng() % 6); + for (unsigned k = 0; k < pieces; ++k) + { + content += valid[rng() % valid.size()]; + content += std::string(k == 0 ? run : rng() % 41, static_cast('b' + (rng() % 20))); + switch (rng() % 8) + { + case 0: + content += utf8[rng() % 3]; + break; + case 1: + content += utf8[rng() % utf8.size()]; + break; + case 2: + content += invalid[rng() % invalid.size()]; + break; + default: + break; + } + } + check_string(content); + check_string(content.substr(0, content.size() - (rng() % 4))); + } + } + // the text before the first escape, and runs and escapes at the end of the text + for (std::size_t before = 0; before <= 40; ++before) + { + for (const char* escape : valid) + { + check_string(std::string(before, 'a') + escape); + check_string(std::string(before, 'a') + escape + std::string(before % 19, 'z')); + check_string(std::string(before, 'a') + escape + escape + std::string(before % 19, 'z')); + } + } +} + TEST_CASE("json_view node integer bits") { using nlohmann::detail::view::integer_bits; diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index 3b9da6b89..f3ccc004e 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -21,6 +21,7 @@ using nlohmann::ordered_json_view; using json_view_test::generator; using json_view_test::has_duplicate_keys; +#include #include #include #include @@ -29,6 +30,7 @@ using json_view_test::has_duplicate_keys; #include #include #include +#include #include #include #include @@ -186,6 +188,133 @@ TEST_CASE("json_view dump") CHECK(deep_document.root().dump() == deep); } + SECTION("ensure_ascii: a character that needs an escape at each position of a string") + { + // string lengths around the width of the vector steps, in a string value, a key, and a value near the end of the text + for (const char* const special : + { + "\xC3\xA9", "\xE4\xB8\xAD", "\xF0\x9F\x98\x80", "\x7F", "\\n", "\\u0001", "\\\"", "\\\\" + }) + { + for (std::size_t length = 0; length <= 40; ++length) + { + for (std::size_t position = 0; position <= length; ++position) + { + std::string content(length, 'a'); + content.insert(position, special); + const std::array texts = + { + { + "\"" + content + "\"", + "[\"" + content + "\"]", + "{\"" + content + "\":1}", + } + }; + for (const std::string& text : texts) + { + CAPTURE(text) + const ordered_json_document d = ordered_json_document::parse(text); + const ordered_json j = ordered_json::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(2, ' ', true) == j.dump(2, ' ', true)); + CHECK(d.root().dump() == j.dump()); + } + } + } + } + } + + SECTION("ensure_ascii: every character of the Basic Multilingual Plane and some beyond") + { + // (the two- and three-byte sequences, including the ranges that need a check: E0, ED, EF BF BF) + std::string text = "["; + std::size_t in_chunk = 0; + const auto append = [&](std::uint32_t cp) + { + if (cp < 0x80) + { + if (cp < 0x20 || cp == '"' || cp == '\\') + { + return; + } + text += static_cast(cp); + } + else if (cp < 0x800) + { + text += static_cast(0xC0 | (cp >> 6)); + text += static_cast(0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + text += static_cast(0xE0 | (cp >> 12)); + text += static_cast(0x80 | ((cp >> 6) & 0x3F)); + text += static_cast(0x80 | (cp & 0x3F)); + } + else + { + text += static_cast(0xF0 | (cp >> 18)); + text += static_cast(0x80 | ((cp >> 12) & 0x3F)); + text += static_cast(0x80 | ((cp >> 6) & 0x3F)); + text += static_cast(0x80 | (cp & 0x3F)); + } + }; + for (std::uint32_t cp = 0x20; cp < 0x10400; ++cp) + { + if (cp >= 0xD800 && cp < 0xE000) + { + continue; + } + if (in_chunk == 0) + { + text += text.size() > 1 ? ",\"" : "\""; + } + append(cp); + if (++in_chunk == 37) + { + text += "\""; + in_chunk = 0; + } + } + text += in_chunk != 0 ? "\"]" : "]"; + const ordered_json_document d = ordered_json_document::parse(text); + const ordered_json j = ordered_json::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(2, ' ', true) == j.dump(2, ' ', true)); + CHECK(d.root().dump() == j.dump()); + CHECK(d.root().dump(2) == j.dump(2)); + } + + SECTION("deep nesting, indented") + { + // indentations of up to and beyond the width of the writer's fixed-size copy + for (const int depth : + { + 1, 5, 20, 70, 300 + }) + { + std::string text; + for (int i = 0; i < depth; ++i) + { + text += i % 2 == 0 ? "[1," : R"({"k":2,"\u00e9":)"; + } + text += "[]"; + for (int i = depth - 1; i >= 0; --i) + { + text += i % 2 == 0 ? "]" : "}"; + } + const ordered_json_document d = ordered_json_document::parse(text); + const ordered_json j = ordered_json::parse(text); + for (const int indent : + { + 0, 1, 2, 3, 8 + }) + { + CHECK(d.root().dump(indent) == j.dump(indent)); + CHECK(d.root().dump(indent, '\t') == j.dump(indent, '\t')); + } + } + } + SECTION("the output buffer of a small value is small") { // an escaped key after the value: its node lies in the arena, so the @@ -598,6 +727,152 @@ TEST_CASE("json_view large objects") CHECK(without_collisions.memory_usage() >= with_collisions.memory_usage() + (table_size * sizeof(std::uint32_t))); } + SECTION("keys that differ in their last bytes are spread over the table") + { + // The hash must spread the usual families of keys (numbered names, hex + // strings, dates, short names): a key farther than index_max_displacement + // from its home slot would leave the object without a table. The keys + // are placed with linear probing, as the table does. + const auto max_displacement = [](const std::vector& keys) + { + std::size_t slots = 16; + while (slots < 2 * keys.size()) + { + slots *= 2; + } + std::vector used(slots, false); + std::size_t farthest = 0; + for (const std::string& key : keys) + { + std::size_t i = nlohmann::detail::conditional_static_cast(nlohmann::detail::view::key_hash(key.data(), key.size())) & (slots - 1); + std::size_t distance = 0; + while (used[i]) + { + i = (i + 1) & (slots - 1); + ++distance; + } + used[i] = true; + farthest = (std::max)(farthest, distance); + } + return farthest; + }; + // the keys of a printf-like pattern with one %d, %0Nd or %0Nx, formatted with a stream + // (a format string that is not a literal is a Flawfinder error) + const auto numbered = [](const char* format, int count, int step) + { + const std::string pattern(format); + const std::size_t percent = pattern.find('%'); + std::size_t conversion = percent + 1; + std::size_t width = 0; + while (pattern[conversion] >= '0' && pattern[conversion] <= '9') + { + width = (width * 10) + static_cast(pattern[conversion] - '0'); + ++conversion; + } + const bool hex = pattern[conversion] == 'x'; + std::vector keys; + for (int i = 0; i < count; ++i) + { + std::ostringstream key; + key << pattern.substr(0, percent) << std::setfill('0') << std::setw(static_cast(width)) + << (hex ? std::hex : std::dec) << (i * step) << pattern.substr(conversion + 1); + keys.push_back(key.str()); + } + return keys; + }; + for (const char* format : + {"key%d", "%d", "item_%04d", "a%05d", "%03dxyz", "id%d", "user_%d", "%d_suffix", "0x%04x", "A%07d", "12345678%d", "1234567%d", "123456789%d", + "configuration_parameter_value_%04d_extra", "2026-01-%02dT00:00:00Z" + }) + { + CAPTURE(format) + CHECK(max_displacement(numbered(format, std::string(format).find("%02d") != std::string::npos ? 60 : 1000, 1)) < nlohmann::detail::view::index_max_displacement); // NOLINT(abseil-string-find-str-contains) + CHECK(max_displacement(numbered(format, 600, 7)) < nlohmann::detail::view::index_max_displacement); + } + std::mt19937_64 rng(7); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + for (const std::size_t length : + { + std::size_t{1}, std::size_t{2}, std::size_t{3}, std::size_t{5}, std::size_t{7}, std::size_t{8}, std::size_t{9}, std::size_t{12}, std::size_t{16}, std::size_t{17}, std::size_t{33} + }) + { + CAPTURE(length) + std::set distinct; + // (there are 26 and 676 keys of one and two letters) + std::size_t wanted = 700; + if (length == 1) + { + wanted = 26; + } + else if (length == 2) + { + wanted = 500; + } + while (distinct.size() < wanted) + { + std::string key(length, 'a'); + for (char& c : key) + { + c = static_cast('a' + (rng() % 26)); + } + distinct.insert(key); + } + CHECK(max_displacement(std::vector(distinct.begin(), distinct.end())) < nlohmann::detail::view::index_max_displacement); + } + } + + SECTION("keys of every length that differ in one byte") + { + // In a large object, a key is found by comparing its bytes with those of the member the + // table points to: up to 16 bytes as two overlapping words, longer ones with memcmp. + // Keys of every length from 0 to 40, and keys that differ from them in one byte. + std::mt19937_64 rng(2026); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) + std::vector keys; + std::set distinct; + for (std::size_t length = 0; length <= 40; ++length) + { + for (int copy = 0; copy < 5; ++copy) + { + std::string key(length, 'a'); + for (char& c : key) + { + c = static_cast('a' + (rng() % 26)); + } + if (distinct.insert(key).second) + { + keys.push_back(key); + } + } + } + std::string text = "{"; + for (std::size_t i = 0; i < keys.size(); ++i) + { + text += (i != 0 ? ",\"" : "\"") + keys[i] + "\":" + std::to_string(i); + } + text += "}"; + REQUIRE(keys.size() >= 128); // (a table is built for 128 members) + const json_document d = json_document::parse(text); + const json_view v = d.root(); + for (std::size_t i = 0; i < keys.size(); ++i) + { + CAPTURE(keys[i]) + CHECK(v[keys[i]].get() == i); + CHECK(v.at(keys[i]).get() == i); + CHECK(v.contains(keys[i])); + for (std::size_t at = 0; at < keys[i].size(); ++at) + { + std::string other = keys[i]; + other[at] = other[at] == 'z' ? 'y' : 'z'; + CAPTURE(other) + CHECK(v.contains(other) == (distinct.count(other) != 0)); + } + CHECK(v.contains(keys[i] + "a") == (distinct.count(keys[i] + "a") != 0)); + if (!keys[i].empty()) + { + CHECK(v.contains(keys[i].substr(1)) == (distinct.count(keys[i].substr(1)) != 0)); + } + } + } + SECTION("shrink_to_fit releases the tables' spare capacity") { const auto make_text = [](int objects, int members) diff --git a/tests/src/unit-json_view_edit.cpp b/tests/src/unit-json_view_edit.cpp index 5c3b4396a..4a6fbe69d 100644 --- a/tests/src/unit-json_view_edit.cpp +++ b/tests/src/unit-json_view_edit.cpp @@ -661,6 +661,115 @@ TEST_CASE("json_view edits: replacing arrays and objects by scalars") CHECK(d.root().materialize() == expected); } +TEST_CASE("json_view edits: arrays whose elements have no subtrees") +{ + // such an array is indexed directly while it is in its parsed layout; edits + // that replace elements by scalars keep it, others move it + json_editable_document d = json_editable_document::parse("[10,20,30,40]"); + json expected = json::array({10, 20, 30, 40}); + const auto check = [&] + { + CHECK(d.root().size() == expected.size()); + for (std::size_t i = 0; i < expected.size(); ++i) + { + CHECK(d.root()[i].materialize() == expected[i]); + CHECK(d.root().at(i).materialize() == expected[i]); + } + CHECK(d.root()[expected.size()].is_discarded()); + CHECK(d.root().back().materialize() == expected.back()); + CHECK(d.root().dump() == expected.dump()); + }; + check(); + d.set(d.root()[1], "x"); + expected[1] = "x"; + check(); + d.set(d.root()[3], 2.5); + expected[3] = 2.5; + check(); + d.set(d.root()[2], json::array({1, 2})); + expected[2] = json::array({1, 2}); + check(); + d.push_back(d.root(), 50); + expected.push_back(50); + check(); + d.insert(d.root(), 0, true); + expected.insert(expected.begin(), true); + check(); + d.erase(d.root(), 1); + expected.erase(expected.begin() + 1); + check(); + d.set(d.root()[2], nullptr); + expected[2] = nullptr; + check(); +} + +TEST_CASE("json_view edits: materializing values nested deeper than the document was parsed") +{ + // the stack of open containers of materialize() is sized from the nesting of the parsed text; + // a value added by an edit can be deeper than that (and than the stack that is local to the call) + json_editable_document d = json_editable_document::parse("[1,[2]]"); + json expected = json::array({1, json::array({2})}); + json nested = 3; + for (int level = 0; level < 40; ++level) + { + nested = json::array({nested, json{{"k", level}}}); + } + d.push_back(d.root(), nested); + expected.push_back(nested); + CHECK(d.root().materialize() == expected); + CHECK(d.root()[2].materialize() == nested); + d.set(d.root()[1][0], nested); + expected[1][0] = nested; + CHECK(d.root().materialize() == expected); + CHECK(d.root().get() == expected); +} + +TEST_CASE("json_view edits: an editable document before and after its first edit") +{ + // until the first edit, an editable document is read like a read-only one; views made before the edit + // keep working after it + const std::string text = R"({"a":[1,2,{"x":"y"},4],"b":{"c":true,"d":null},"e":"text","f":[]})"; + const json_document read_only = json_document::parse(text); + json_editable_document d = json_editable_document::parse(text); + const auto root = d.root(); + const auto same_as_read_only = [&] + { + CHECK(root.dump() == read_only.root().dump()); + CHECK(root.dump(2) == read_only.root().dump(2)); + CHECK(root.materialize() == read_only.root().materialize()); + CHECK(root["a"][2]["x"].get_string() == "y"); + CHECK(root["a"].at(3).get() == 4); + CHECK(root["a"].back().get() == 4); + CHECK(root.contains("b")); + CHECK(!root.contains("zz")); + CHECK(root.find("e")->get_string() == "text"); + CHECK(root.find("zz") == root.end()); + CHECK(root["b"]["c"].get()); + CHECK(root.value("e", "none") == "text"); + std::size_t count = 0; + for (const auto m : root["a"]) + { + CHECK(m.materialize() == read_only.root()["a"][count].materialize()); + ++count; + } + CHECK(count == 4); + }; + same_as_read_only(); + // an edit that changes a scalar in place, then one that moves an array + d.set(root["e"], "other"); + CHECK(root["e"].get_string() == "other"); + CHECK(root["a"][2]["x"].get_string() == "y"); + CHECK(root["a"].back().get() == 4); + d.push_back(root["f"], 7); + CHECK(root["f"][0].get() == 7); + CHECK(root["f"].back().get() == 7); + CHECK(root.dump() == R"({"a":[1,2,{"x":"y"},4],"b":{"c":true,"d":null},"e":"other","f":[7]})"); + d.set(root["a"], 2, json::array({1, 2})); + CHECK(root["a"][2][1].get() == 2); + CHECK(root["a"].dump() == "[1,2,[1,2],4]"); + CHECK(root.materialize() == json::parse(R"({"a":[1,2,[1,2],4],"b":{"c":true,"d":null},"e":"other","f":[7]})")); +} + namespace { // reads of "a" in an object that repeats it: the first member wins, however the diff --git a/tests/src/unit-json_view_image.cpp b/tests/src/unit-json_view_image.cpp index 50023fe63..eaf27eb9f 100644 --- a/tests/src/unit-json_view_image.cpp +++ b/tests/src/unit-json_view_image.cpp @@ -682,6 +682,26 @@ TEST_CASE("json_view images: ownership") #if !defined(JSON_NOEXCEPTION) TEST_CASE("json_view images: errors") { + SECTION("invalid UTF-8 of a decoded string: the compact dump writes it, the others throw") + { + const json_document d = json_document::parse(R"({"k\n":"\u00e9x"})"); + std::vector image = d.save(); + const std::size_t arena = text_at(image) + header_size_field(image, 16) + 1; + image[arena + 3] = 0x28; // the key is "k\n"; the value starts at 3: 0xC3 0x28 is invalid + const json_document loaded = json_document::load(image, image_check::bounds); + CHECK(loaded.root().dump().size() == 13); + for (const int indent : + { + 0, 2 + }) + { + CHECK_THROWS_AS(loaded.root().dump(indent), json::type_error); + CHECK_THROWS_AS(loaded.root().dump(indent, ' ', true), json::type_error); + } + CHECK_THROWS_AS(loaded.root().dump(-1, ' ', true), json::type_error); + CHECK_THROWS_WITH_AS(loaded.root().dump(2), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0x28", json::type_error&); + } + SECTION("a literal as the root: dump() after loading") { for (const char* text : diff --git a/tests/src/unit-ordered_json.cpp b/tests/src/unit-ordered_json.cpp index 135dedade..c328d8aea 100644 --- a/tests/src/unit-ordered_json.cpp +++ b/tests/src/unit-ordered_json.cpp @@ -12,6 +12,11 @@ using nlohmann::json; using nlohmann::ordered_json; +#include +#include +#include +#include + TEST_CASE("ordered_json") { json j; @@ -237,3 +242,276 @@ TEST_CASE("regression test for issue #5673 - ordered_json::emplace with a non-rv CHECK(oj.dump() == "{\"a\":1}"); } } + +namespace +{ +// the sizes around which the DOM parser changes how it stores the elements of +// an array (16) and the members of an object in a vector (32, and 64 for the +// key index) +const std::array container_sizes = {{0, 1, 2, 3, 15, 16, 17, 18, 31, 32, 33, 34, 63, 64, 65, 66, 200}}; + +template +std::string array_text(const std::size_t n, const std::size_t depth) +{ + std::string text = "["; + for (std::size_t i = 0; i < n; ++i) + { + text += (i == 0 ? "" : ","); + // alternate scalars, empty containers and nested containers + switch (i % 4) + { + case 0: + text += std::to_string(i); + break; + case 1: + text += "\"s" + std::to_string(i) + "\""; + break; + case 2: + text += depth == 0 ? "[]" : array_text(n / 2, depth - 1); + break; + default: + text += depth == 0 ? "{}" : "{\"a\":" + array_text(n / 3, depth - 1) + ",\"b\":null}"; + break; + } + } + return text + "]"; +} + +template +std::string object_text(const std::size_t n, const std::size_t depth, const bool duplicates) +{ + std::string text = "{"; + for (std::size_t i = 0; i < n; ++i) + { + text += (i == 0 ? "" : ","); + text += "\"key" + std::to_string(i) + "\":"; + switch (i % 4) + { + case 0: + text += std::to_string(i); + break; + case 1: + text += depth == 0 ? "[]" : array_text(n / 2, depth - 1); + break; + case 2: + text += depth == 0 ? "{}" : object_text(n / 2, depth - 1, duplicates); + break; + default: + text += "\"value\""; + break; + } + } + if (duplicates) + { + // keys that are there already, the first and the last one, with other values + text += n == 0 ? "" : ","; + text += R"("key0":[1,2],"key)"; + text += std::to_string(n - 1); + text += R"(":{"again":true})"; + } + return text + "}"; +} + +template +void check_same_dump(const std::string& text) +{ + // the text is what dump() writes for the parsed value, and the value + // survives a second round trip: arrays keep their elements in order, an + // ordered_json keeps its members in order of appearance + const Json parsed = Json::parse(text); + const Json again = Json::parse(parsed.dump()); + CHECK(parsed == again); + CHECK(parsed.dump() == again.dump()); + CHECK(Json::accept(text)); +} +} // namespace + +TEST_CASE("parsing containers of sizes around the limits of the DOM parser") +{ + SECTION("arrays") + { + for (const std::size_t n : container_sizes) + { + const std::string text = array_text(n, 2); + INFO("size ", n); + + const json parsed = json::parse(text); + CHECK(parsed.is_array()); + CHECK(parsed.size() == n); + for (std::size_t i = 0; i < n; i += 4) + { + CHECK(parsed[i] == i); + } + check_same_dump(text); + check_same_dump(text); + CHECK(ordered_json::parse(text).dump() == parsed.dump()); + } + } + + SECTION("objects") + { + for (const bool duplicates : + { + false, true + }) + { + for (const std::size_t n : container_sizes) + { + const std::string text = object_text(n, 2, duplicates); + INFO("size ", n, " duplicates ", duplicates); + + const json parsed = json::parse(text); + const ordered_json ordered = ordered_json::parse(text); + CHECK(parsed.is_object()); + CHECK(ordered.is_object()); + CHECK(parsed.size() == ordered.size()); + // the same members, in the order of their first appearance + std::size_t i = 0; + for (auto it = ordered.begin(); it != ordered.end(); ++it, ++i) + { + if (i < n) + { + CHECK(it.key() == "key" + std::to_string(i)); + } + CHECK(parsed.at(it.key()) == json(it.value())); + } + if (duplicates && n > 1) + { + // the later value replaced the earlier one in its place + CHECK(ordered.at("key0") == ordered_json::parse("[1,2]")); + CHECK(ordered.at("key" + std::to_string(n - 1)) == ordered_json::parse("{\"again\":true}")); + CHECK(ordered.begin().key() == "key0"); + CHECK(ordered.size() == n); + } + check_same_dump(text); + check_same_dump(text); + } + } + } + + SECTION("objects with many different and repeated keys") + { + // 300 members of which a third repeat an earlier key + std::string text = "{"; + for (std::size_t i = 0; i < 300; ++i) + { + text += (i == 0 ? "" : ","); + text += "\"k" + std::to_string(i % 200) + "\":" + std::to_string(i); + } + text += '}'; + + const ordered_json ordered = ordered_json::parse(text); + const json parsed = json::parse(text); + CHECK(ordered.size() == 200); + CHECK(parsed.size() == 200); + std::size_t i = 0; + for (auto it = ordered.begin(); it != ordered.end(); ++it, ++i) + { + CHECK(it.key() == "k" + std::to_string(i)); + // the last value written + CHECK(it.value() == (i < 100 ? i + 200 : i)); + CHECK(parsed.at(it.key()) == json(it.value())); + } + } +} + +namespace +{ +// a deterministic source of structure for parsing tests: builds a JSON text +// and, at the same time, the value it has to parse to +class tree_generator +{ + public: + explicit tree_generator(std::uint32_t seed) : state(seed) {} + + template + Json make(std::string& text, const std::size_t depth) + { + switch (next(depth == 0 ? 5 : 9)) + { + case 0: + text += "null"; + return Json(nullptr); + case 1: + text += "true"; + return Json(true); + case 2: + { + const int n = static_cast(next(2000)) - 1000; + text += std::to_string(n); + return Json(n); + } + case 3: + text += "\"text\""; + return Json("text"); + case 4: + text += "2.5"; + return Json(2.5); + case 5: + case 6: + case 7: + { + Json array = Json::array(); + text += '['; + const std::size_t n = size(); + for (std::size_t i = 0; i < n; ++i) + { + text += (i == 0 ? "" : ","); + array.push_back(make(text, depth - 1)); + } + text += ']'; + return array; + } + default: + { + Json object = Json::object(); + text += '{'; + const std::size_t n = size(); + for (std::size_t i = 0; i < n; ++i) + { + const std::string key = "k" + std::to_string(i); + text += (i == 0 ? "\"" : ",\"") + key + "\":"; + object[key] = make(text, depth - 1); + } + text += '}'; + return object; + } + } + } + + private: + std::uint32_t next(const std::uint32_t bound) + { + state = (state * 1664525U) + 1013904223U; + return (state >> 8) % bound; + } + + std::size_t size() + { + return container_sizes[next(static_cast(container_sizes.size()))]; + } + + std::uint32_t state; +}; +} // namespace + +TEST_CASE("parsing random trees of arrays and objects") +{ + for (std::uint32_t seed = 1; seed <= 150; ++seed) + { + INFO("seed ", seed); + + std::string text; + std::string ordered_text; + const json expected = tree_generator(seed).make(text, 3); + const auto ordered_expected = tree_generator(seed).make(ordered_text, 3); + CHECK(text == ordered_text); + + CHECK(json::parse(text) == expected); + CHECK(json::accept(text)); + const ordered_json ordered = ordered_json::parse(text); + CHECK(ordered == ordered_expected); + CHECK(ordered.dump() == ordered_expected.dump()); + CHECK(json::parse(text).dump() == expected.dump()); + } +} diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index 5722c75a0..1173b851b 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -1580,3 +1580,94 @@ TEST_CASE("invalid UTF-8 at every position is handled like the reference") } } } + +TEST_CASE("short strings are written like the reference, as values and as keys") +{ + // strings of up to 16 characters (32 with vector instructions) that need no + // escaping are written as a whole (together with the quotes and, for a key, + // the colon); everything else, including a string that is merely short, + // takes the general path. + // Every length around the edges of the size classes of the fast path is + // combined with each kind of byte that must send a string to the general + // path, at each position, and compared with the reference for values and, + // via the value, for keys. + const std::vector specials = + { + "\"", "\\", "\n", "\x1f", "\x7f", "\xc3\xa9", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xff", "\x80", "\xc3" + }; + + for (std::size_t length = 0; length <= 36; ++length) + { + const std::string plain(length, 'k'); + check_dump_against_reference(plain, false); + + for (const auto& special : specials) + { + for (std::size_t pos = 0; pos <= length; ++pos) + { + std::string s = plain; + s.insert(pos, special); + check_dump_against_reference(s, false); + + for (const bool ensure_ascii : + { + false, true + }) + { + // a key and a value are written alike, in compact and + // in pretty-printed form + const json value = s; + const json object = json::object({{s, 1}}); + for (const auto error_handler : + { + json::error_handler_t::replace, json::error_handler_t::keep + }) + { + const std::string quoted = value.dump(-1, ' ', ensure_ascii, error_handler); + CHECK(object.dump(-1, ' ', ensure_ascii, error_handler) == "{" + quoted + ":1}"); + CHECK(object.dump(2, ' ', ensure_ascii, error_handler) == "{\n " + quoted + ": 1\n}"); + } + } + } + } + } +} + +TEST_CASE("runs of multi-byte characters are escaped like the reference under ensure_ascii") +{ + // a run of well-formed multi-byte characters is escaped straight into the + // write buffer; it has to flush that mid-run, stop at an ASCII character or + // an ill-formed sequence and leave those to the general path, and pick up + // behind what the general path has pending + const std::vector characters = + { + "\xc2\x80", "\xdf\xbf", "\xe0\xa0\x80", "\xe2\x82\xac", "\xed\x9f\xbf", "\xef\xbf\xbf", + "\xf0\x90\x80\x80", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf" + }; + const std::vector interruptions = + { + "", "a", "\n", "\"", "\x7f", "\xff", "\x80", "\xc3", "\xe2\x82", "\xf0\x9f\x98", "\xed\xa0\x80" + }; + + for (std::size_t shift = 0; shift <= 13; ++shift) + { + for (const auto& interruption : interruptions) + { + for (const std::size_t count : + { + std::size_t{1}, std::size_t{2}, std::size_t{85}, std::size_t{200} + }) + { + std::string s(shift, 'x'); + for (std::size_t i = 0; i < count; ++i) + { + s += characters[(i + shift) % characters.size()]; + } + s += interruption; + s += characters[shift % characters.size()]; + s += characters[(shift + 3) % characters.size()]; + check_dump_against_reference(s, shift < 2); + } + } + } +} diff --git a/tests/src/unit-stl_helpers.cpp b/tests/src/unit-stl_helpers.cpp index 05c1e21bf..3d00236f8 100644 --- a/tests/src/unit-stl_helpers.cpp +++ b/tests/src/unit-stl_helpers.cpp @@ -78,7 +78,8 @@ TEST_CASE("detail::bit_cast") // to and from byte arrays const auto bytes = bit_cast>(std::uint32_t{0x01020304u}); - CHECK(bit_cast(bytes) == 0x01020304u); + // qualified: with a std::array argument, ADL would also find std::bit_cast in C++20 + CHECK(nlohmann::detail::bit_cast(bytes) == 0x01020304u); static_assert(noexcept(bit_cast(1.0)), "bit_cast must be noexcept"); } diff --git a/tests/src/unit-stream-bulk-scan.cpp b/tests/src/unit-stream-bulk-scan.cpp index 14e61d6f5..d61ce4bd0 100644 --- a/tests/src/unit-stream-bulk-scan.cpp +++ b/tests/src/unit-stream-bulk-scan.cpp @@ -48,14 +48,16 @@ namespace class chunked_streambuf : public std::streambuf { public: - chunked_streambuf(std::string s, std::size_t chunk_size) - : m_data(std::move(s)), m_chunk_size(chunk_size) + // The first @a unbuffered characters are served without a get area, the + // rest in get areas of @a chunk_size. + chunked_streambuf(std::string s, std::size_t chunk_size, std::size_t unbuffered = 0) + : m_data(std::move(s)), m_chunk_size(chunk_size), m_unbuffered(unbuffered) {} protected: int_type underflow() override { - if (m_chunk_size == 0) + if (m_chunk_size == 0 || m_pos < m_unbuffered) { return m_pos < m_data.size() ? traits_type::to_int_type(m_data[m_pos]) : traits_type::eof(); } @@ -77,7 +79,7 @@ class chunked_streambuf : public std::streambuf int_type uflow() override { - if (m_chunk_size == 0) + if (m_chunk_size == 0 || m_pos < m_unbuffered) { return m_pos < m_data.size() ? traits_type::to_int_type(m_data[m_pos++]) : traits_type::eof(); } @@ -88,6 +90,7 @@ class chunked_streambuf : public std::streambuf std::string m_data; std::size_t m_pos = 0; std::size_t m_chunk_size; + std::size_t m_unbuffered; }; // how a test reads from the stream @@ -101,9 +104,9 @@ enum class read_mode // Read @a input from a chunked_streambuf and describe everything observable: // the values or the exception, the stream state after each value, and the // characters left in the stream. -std::string observe(const std::string& input, std::size_t chunk_size, read_mode mode) +std::string observe(const std::string& input, std::size_t chunk_size, read_mode mode, std::size_t unbuffered = 0) { - chunked_streambuf buf(input, chunk_size); + chunked_streambuf buf(input, chunk_size, unbuffered); std::istream is(&buf); std::string result; for (int i = 0; i < 4; ++i) @@ -212,6 +215,31 @@ TEST_CASE("parsing from a std::istream scans the get area in bulk") } } + SECTION("results do not depend on a get area that appears late") + { + // the first characters come without a get area, then the streambuf + // starts to use one: the token that is read at that point is complete + for (const auto& input : corpus()) + { + const std::string expected = observe(input, 0, read_mode::extract); + for (const std::size_t unbuffered : + { + 1u, 2u, 3u, 5u, 8u, 13u, 21u + }) + { + for (const std::size_t chunk_size : + { + 1u, 2u, 3u, 7u, 64u + }) + { + CAPTURE(input) + CAPTURE(unbuffered) + CAPTURE(chunk_size) + CHECK(observe(input, chunk_size, read_mode::extract, unbuffered) == expected); + } + } + } + } #endif // !defined(JSON_NOEXCEPTION) SECTION("values are the same as when parsing from a string") diff --git a/tools/generate_natvis/generate_natvis.py b/tools/generate_natvis/generate_natvis.py index dc19ec01c..d660c4b63 100755 --- a/tools/generate_natvis/generate_natvis.py +++ b/tools/generate_natvis/generate_natvis.py @@ -139,9 +139,12 @@ if __name__ == '__main__': namespaces += [f'{namespaces[0]}::{ns}' for ns in inline_namespaces] + # The template renders a local XML file, not a web response, and autoescape is enabled. + # nosemgrep: python.flask.security.xss.audit.direct-use-of-jinja2.direct-use-of-jinja2 env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0]), autoescape=True, trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True) template = env.get_template('nlohmann_json.natvis.j2') + # nosemgrep: python.flask.security.xss.audit.direct-use-of-jinja2.direct-use-of-jinja2 natvis = template.render(namespaces=namespaces) output = args.output or os.path.join(args.repo_root, 'nlohmann_json.natvis')