Compare commits

..
Author SHA1 Message Date
Niels Lohmann db1d0983e6 create objects for tokens that cannot be array indices (#5357)
When operator[](json_pointer) traverses a level that does not exist yet,
the null value is turned into an array or an object depending on the
reference token. The check only tested whether all characters are digits,
so tokens that can never be a valid array index selected an array and
then failed:

- "01" (and any other token with a leading '0') threw parse_error.106
- the empty token threw out_of_range.404

Both tokens are valid object keys, and both work when the level already
exists as an object, so creating the level changed the outcome. Test the
token against the RFC 6901, Sect. 4 grammar for array indices instead, so
that such tokens create an object. This only affects pointers that threw
before.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-08-04 08:51:54 +02:00
25 changed files with 153 additions and 334 deletions
+3 -3
View File
@@ -38,14 +38,14 @@ jobs:
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
with:
languages: c-cpp
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
uses: github/codeql-action/autobuild@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
+1 -1
View File
@@ -43,6 +43,6 @@ jobs:
output: 'flawfinder_results.sarif'
- name: Upload analysis results to GitHub Security tab
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
with:
sarif_file: ${{github.workspace}}/flawfinder_results.sarif
+1 -1
View File
@@ -76,6 +76,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
with:
sarif_file: results.sarif
+1 -1
View File
@@ -61,7 +61,7 @@ jobs:
# Upload SARIF file generated in previous step
- name: Upload SARIF file
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
uses: github/codeql-action/upload-sarif@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3
with:
sarif_file: semgrep.sarif
if: always()
+17 -17
View File
@@ -25,7 +25,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -47,7 +47,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -70,7 +70,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -89,7 +89,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -108,7 +108,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -184,7 +184,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: CXX=g++-${{ matrix.compiler }} cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -202,7 +202,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -212,14 +212,14 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
compiler: ['3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15-bullseye', '16', '17', '18', '19', '20', '21', '22', 'latest']
compiler: ['3.4', '3.5', '3.6', '3.7', '3.8', '3.9', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15-bullseye', '16', '17', '18', '19', '20', 'latest']
container: silkeh/clang:${{ matrix.compiler }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Set env FORCE_STDCPPFS_FLAG for clang 7 / 8 / 9 / 10
run: echo "JSON_FORCED_GLOBAL_COMPILE_OPTIONS=-DJSON_HAS_FILESYSTEM=0;-DJSON_HAS_EXPERIMENTAL_FILESYSTEM=0" >> "$GITHUB_ENV"
if: ${{ matrix.compiler == '7' || matrix.compiler == '8' || matrix.compiler == '9' || matrix.compiler == '10' }}
@@ -239,7 +239,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -259,7 +259,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build with libc++
@@ -286,7 +286,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -306,7 +306,7 @@ jobs:
# import-std support. Its opt-in token is CMake-version-specific, so pin
# CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt.
- name: Get pinned CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
with:
cmakeVersion: 4.3.4
# Clang: the std library module is provided by libc++ (the image's libstdc++
@@ -332,7 +332,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -347,7 +347,7 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -359,7 +359,7 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
@@ -379,7 +379,7 @@ jobs:
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Run CMake
run: cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -GNinja
- name: Build
+2 -8
View File
@@ -88,7 +88,7 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Get latest CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
- name: Set extra CXX_FLAGS for latest std_version
# /wd5285 silences C5285 emitted by the bundled third-party doctest.h, which
# specializes std::tuple (newly diagnosed by the VS2026 v145 toolset)
@@ -153,16 +153,10 @@ jobs:
with:
platform: x64
version: 12.2.0 # https://github.com/egor-tensin/setup-mingw/issues/14
# CMAKE_CXX_FLAGS_DEBUG is overridden to drop the default -g: linking
# test-regression2_cpp20 intermittently fails with "relocation truncated
# to fit: IMAGE_REL_AMD64_SECREL against `.debug_line'" because the
# MinGW linker cannot relocate the debug sections this test produces.
# The tests are only built and run here, so the debug info is not used.
- name: Run CMake
run: cmake -S . -B build ^
-DCMAKE_CXX_COMPILER="C:/Program Files/LLVM/bin/clang++.exe" ^
-DCMAKE_CXX_FLAGS="--target=x86_64-w64-mingw32 -stdlib=libstdc++ -pthread" ^
-DCMAKE_CXX_FLAGS_DEBUG="-g0" ^
-DCMAKE_EXE_LINKER_FLAGS="-lwinpthread" ^
-G"MinGW Makefiles" ^
-DCMAKE_BUILD_TYPE=Debug ^
@@ -199,7 +193,7 @@ jobs:
# import-std support. Its opt-in token is CMake-version-specific, so pin
# CMake to the version whose token is set in tests/module_cpp20/CMakeLists.txt.
- name: Get pinned CMake and ninja
uses: lukka/get-cmake@4a7d025fc60f00db0c7b44ebf783d19b52444830 # v4.4.1
uses: lukka/get-cmake@e6906078ebd1ccb8ce51ab4626ac46a1b5a517e3 # v4.4.0
with:
cmakeVersion: 4.3.4
- name: Run CMake (Debug)
-6
View File
@@ -42,7 +42,6 @@
- [Specializing enum conversion](#specializing-enum-conversion)
- [Binary formats (BSON, CBOR, MessagePack, UBJSON, and BJData)](#binary-formats-bson-cbor-messagepack-ubjson-and-bjdata)
- [Customers](#customers)
- [Ecosystem](#ecosystem)
- [Supported compilers](#supported-compilers)
- [Integration](#integration)
- [CMake](#cmake)
@@ -1187,11 +1186,6 @@ The library is used in multiple projects, applications, operating systems, etc.
[![logos of customers using the library](docs/mkdocs/docs/images/customers.png)](https://json.nlohmann.me/home/customers/)
## Ecosystem
Beyond projects that use the library, there are third-party projects that build on top of it - schema validators,
language bindings, format converters, and the like. See the curated [Ecosystem](https://json.nlohmann.me/community/ecosystem/) page.
## Supported compilers
Though it's 2026 already, the support for C++11 is still a bit sparse. Currently, the following compilers are known to work:
@@ -128,10 +128,13 @@ Strong exception safety: if an exception occurs, the original value stays intact
When the JSON pointer traverses intermediate levels that don't exist at all yet (not just a missing
leaf), each missing level is created as an array or an object depending on whether the corresponding
pointer token parses as a non-negative integer: a numeric token creates an array, a non-numeric token
creates an object. For example, on an initially `#!json null` value, `/foo/0/0/0` creates nested arrays,
while `/foo/one/one/one` creates nested objects. This is not specified by the JSON Pointer RFC; it is
this library's own, intentional disambiguation rule. See also [JSON Pointer](../../features/json_pointer.md).
pointer token is a valid array index: a token that is a nonempty sequence of digits without a leading
`0` (or the token `-`) creates an array, and every other token creates an object. For example, on an
initially `#!json null` value, `/foo/0/0/0` creates nested arrays, while `/foo/one/one/one` creates
nested objects. Tokens such as `01` or the empty token cannot be array indices (cf. RFC 6901, Sect. 4)
and therefore create objects, just as they would if the level already existed as an object. This is not
specified by the JSON Pointer RFC; it is this library's own, intentional disambiguation rule. See also
[JSON Pointer](../../features/json_pointer.md).
## Examples
-40
View File
@@ -1,40 +0,0 @@
# Ecosystem
The projects below build on top of `nlohmann::json` rather than merely using it - schema validators, language
bindings, format converters, and similar building blocks. The list is not exhaustive, and is curated rather than
automatically generated. If you maintain or know of a project that belongs here,
[please let me know](mailto:mail@nlohmann.me).
For products, applications, and organizations that use the library, see [Customers](../home/customers.md) instead.
## Schema validation
- [**json-schema-validator**](https://github.com/pboettch/json-schema-validator), a JSON Schema (draft 7) validator
with human-readable error messages
## Serialization and reflection
- [**nlohmann_json_reflect**](https://github.com/1261385937/nlohmann_json_reflect), a reflection extension for
(de)serializing nested containers-in-structs-in-containers
## Encodings
- [**base-encode-decode**](https://github.com/saxonnicholls/base-encode-decode), a header-only Base64/32/16/8/4/2
(and DNA/RNA) encoding library, with an adapter that serializes binary data through `nlohmann::json`
## Language bindings and interop
- [**pybind11_json**](https://github.com/pybind/pybind11_json), a bidirectional type caster between
`nlohmann::json` and Python objects for [pybind11](https://github.com/pybind/pybind11) bindings
- [**nanobind_json**](https://github.com/ianhbell/nanobind_json), the same idea for
[nanobind](https://github.com/wjakob/nanobind) bindings
- [**nlohmann_json_qt**](https://github.com/dpurgin/nlohmann_json_qt), deserialization helpers for Qt types
(`QString`, `QUrl`, `QDateTime`, `QVector`, ...) from `nlohmann::json`
- [**vulkan2json**](https://github.com/Fadis/vulkan2json), serialization and deserialization of Vulkan API structs
## Format converters
- [**tojson**](https://github.com/mircodz/tojson), a header-only converter between YAML/XML documents and
`nlohmann::json`
- [**json2xml**](https://github.com/testillano/json2xml), a header-only converter from `nlohmann::json` to XML for
simple configuration documents
-1
View File
@@ -1,6 +1,5 @@
# Community
- [Ecosystem](ecosystem.md) - third-party projects built on top of this library
- [Code of Conduct](code_of_conduct.md) - the rules and norms of this project
- [Contribution Guidelines](contribution_guidelines.md) - guidelines how to contribute to this project
- [Governance](governance.md) - the governance model of this project
@@ -66,7 +66,6 @@ Note: Some modern features (like C++20 ranges or filesystem support) may be disa
| Clang 20.1.1 | x86_64 | Ubuntu 22.04.1 LTS | GitHub |
| Clang 20.1.8 with GNU-like command-line | x86_64 | Windows Server 2022 (Build 20348) | GitHub |
| Clang 21.1.8 | x86_64 | Ubuntu 22.04.1 LTS | GitHub |
| Clang 22.1.8 | x86_64 | Ubuntu 22.04.1 LTS | GitHub |
| CUDA 11.8.0 (nvcc) | x86_64 | Ubuntu 22.04 LTS | GitHub |
| CUDA 12.1.1 (nvcc) | x86_64 | Ubuntu 22.04 LTS | GitHub |
| CUDA 12.6.3 (nvcc) | x86_64 | Ubuntu 22.04 LTS | GitHub |
@@ -116,19 +116,9 @@ The library uses the following mapping from JSON values types to BJData types ac
```
Likewise, when a JSON object in the above form is serialized using
[`to_bjdata`](../../api/basic_json/to_bjdata.md), it is automatically converted into a compact BJData ND-array. When
the 1-dimensional vector stored in `"_ArraySize_"` contains a single integer or two integers with one being 1, a
regular 1-D optimized array is generated instead.
An object is only converted if the annotation actually describes a packed array; otherwise it is serialized as a
regular JSON object. This requires all of the following:
- `"_ArrayType_"` is one of `uint8`, `int8`, `uint16`, `int16`, `uint32`, `int32`, `uint64`, `int64`, `single`,
`double`, `char`, or `byte`,
- every entry of `"_ArraySize_"` is a non-negative integer, and their product is representable as a `std::size_t`,
- `"_ArrayData_"` holds exactly that many elements, and
- every element of `"_ArrayData_"` is a number of the kind named by `"_ArrayType_"` (a floating-point number for
`single` and `double`, an integer otherwise).
[`to_bjdata`](../../api/basic_json/to_bjdata.md), it is automatically converted into a compact BJData ND-array. The
only exception is, that when the 1-dimensional vector stored in `"_ArraySize_"` contains a single integer or two
integers with one being 1, a regular 1-D optimized array is generated.
The current version of this library does not yet support automatic detection of and conversion from a nested JSON
array input to a BJData ND-array.
-1
View File
@@ -308,7 +308,6 @@ nav:
- 'NLOHMANN_JSON_VERSION_MAJOR, NLOHMANN_JSON_VERSION_MINOR, NLOHMANN_JSON_VERSION_PATCH': api/macros/nlohmann_json_version_major.md
- Community:
- community/index.md
- community/ecosystem.md
- "Code of Conduct": community/code_of_conduct.md
- community/contribution_guidelines.md
- community/quality_assurance.md
+10 -13
View File
@@ -465,6 +465,15 @@ class binary_reader
// CBOR //
//////////
/*!
@param[in] get_char whether a new character should be retrieved from the
input (true) or whether the last read character should
be considered instead (false)
@param[in] tag_handler how CBOR tags should be treated
@return whether a valid CBOR value was passed to the SAX parser
*/
template<typename NumberType>
bool get_cbor_negative_integer()
{
@@ -483,14 +492,6 @@ class binary_reader
return sax->number_integer(static_cast<number_integer_t>(-1) - static_cast<number_integer_t>(number));
}
/*!
@param[in] get_char whether a new character should be retrieved from the
input (true) or whether the last read character should
be considered instead (false)
@param[in] tag_handler how CBOR tags should be treated
@return whether a valid CBOR value was passed to the SAX parser
*/
bool parse_cbor_internal(const bool get_char,
const cbor_tag_handler_t tag_handler)
{
@@ -1987,11 +1988,7 @@ class binary_reader
{
if (get_char)
{
// no get_ignore_noop() here: the byte read next must be a string
// length type specification, and a no-op ('N') is not valid in
// that position. No-ops at positions where a value may appear are
// already consumed by the callers via get_ignore_noop().
get();
get(); // TODO(niels): may we ignore N here?
}
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value")))
@@ -345,12 +345,8 @@ struct wide_string_input_helper<BaseInputAdapter, 4>
}
else
{
// A code point above U+10FFFF has no UTF-8 encoding. Passing the
// unit through would narrow it to int, where 0xFFFFFFFF becomes
// char_traits<char>::eof() and would end the input silently, so
// emit a byte that is never valid UTF-8 and let the decoder
// reject it.
utf8_bytes[0] = 0xFF;
// unknown character
utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
utf8_bytes_filled = 1;
}
}
+4 -8
View File
@@ -370,10 +370,8 @@ class json_sax_dom_parser
case value_t::string:
{
// escape sequences make the token longer than the value it
// parses to, so the start position cannot be derived from
// the value; use the offset the lexer recorded instead
v.start_position = m_lexer_ref->get_token_start_position();
// include the length of the quotes, which is 2
v.start_position = v.end_position - v.m_data.m_value.string->size() - 2;
break;
}
@@ -771,10 +769,8 @@ class json_sax_dom_callback_parser
case value_t::string:
{
// escape sequences make the token longer than the value it
// parses to, so the start position cannot be derived from
// the value; use the offset the lexer recorded instead
v.start_position = m_lexer_ref->get_token_start_position();
// include the length of the quotes, which is 2
v.start_position = v.end_position - v.m_data.m_value.string->size() - 2;
break;
}
-20
View File
@@ -1357,11 +1357,6 @@ scan_number_done:
token_buffer.clear();
decimal_point_position = std::string::npos;
#if JSON_DIAGNOSTIC_POSITIONS
// the first character of the token has already been read, hence the -1
token_start_position = position.chars_read_total - 1;
#endif
note_token_start(std::integral_constant<bool, lazy_token_string> {});
}
@@ -1524,15 +1519,6 @@ scan_number_done:
return position;
}
#if JSON_DIAGNOSTIC_POSITIONS
/// return the offset of the first character of the last read token; unlike
/// the token's parsed value, this accounts for escape sequences
constexpr std::size_t get_token_start_position() const noexcept
{
return token_start_position;
}
#endif
/// seekable adapter: rebuild the last read token from the input on demand
const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::true_type /*lazy*/) const
{
@@ -1733,12 +1719,6 @@ scan_number_done:
/// the last read token on error for seekable adapters (see collect_token_chars)
std::size_t token_string_start = 0;
#if JSON_DIAGNOSTIC_POSITIONS
/// start offset of the current token within the input, used to report
/// diagnostic positions (see reset())
std::size_t token_start_position = 0;
#endif
/// buffer for variable-length tokens (numbers, strings)
string_t token_buffer {};
+9 -5
View File
@@ -396,15 +396,19 @@ class json_pointer
// convert null values to arrays or objects before continuing
if (ptr->is_null())
{
// check if the reference token is a number
const bool nums =
std::all_of(reference_token.begin(), reference_token.end(),
[](const unsigned char x)
// check if the reference token is a valid array index, that is
// a nonempty sequence of digits without a leading '0'
// (cf. RFC 6901, Sect. 4); tokens that could never be a valid
// array index (such as "01" or "") are treated as object keys
const bool nums = !reference_token.empty()
&& (reference_token.size() == 1 || reference_token[0] != '0')
&& std::all_of(reference_token.begin(), reference_token.end(),
[](const unsigned char x)
{
return std::isdigit(x);
});
// change value to an array for numbers or "-" or to object otherwise
// change value to an array for array indices or "-" or to object otherwise
*ptr = (nums || reference_token == "-")
? detail::value_t::array
: detail::value_t::object;
@@ -1670,23 +1670,7 @@ class binary_writer
{
return true;
}
// a dimension that does not fit into std::size_t, or a product that
// overflows it, would wrap around and could match the size of
// _ArrayData_ by accident; the resulting header announces an
// element count that no reader can honor (the binary reader rejects
// it with out_of_range.408), so encode as a plain object instead
const auto dim = el.template get<std::uint64_t>();
if (!value_in_range_of<std::size_t>(dim))
{
return true;
}
const auto dim_size = static_cast<std::size_t>(dim);
if (dim_size != 0 && len > (std::numeric_limits<std::size_t>::max)() / dim_size)
{
return true;
}
len *= dim_size;
len *= static_cast<std::size_t>(el.template get<std::uint64_t>());
}
key = "_ArrayData_";
+26 -69
View File
@@ -7348,12 +7348,8 @@ struct wide_string_input_helper<BaseInputAdapter, 4>
}
else
{
// A code point above U+10FFFF has no UTF-8 encoding. Passing the
// unit through would narrow it to int, where 0xFFFFFFFF becomes
// char_traits<char>::eof() and would end the input silently, so
// emit a byte that is never valid UTF-8 and let the decoder
// reject it.
utf8_bytes[0] = 0xFF;
// unknown character
utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
utf8_bytes_filled = 1;
}
}
@@ -9075,11 +9071,6 @@ scan_number_done:
token_buffer.clear();
decimal_point_position = std::string::npos;
#if JSON_DIAGNOSTIC_POSITIONS
// the first character of the token has already been read, hence the -1
token_start_position = position.chars_read_total - 1;
#endif
note_token_start(std::integral_constant<bool, lazy_token_string> {});
}
@@ -9242,15 +9233,6 @@ scan_number_done:
return position;
}
#if JSON_DIAGNOSTIC_POSITIONS
/// return the offset of the first character of the last read token; unlike
/// the token's parsed value, this accounts for escape sequences
constexpr std::size_t get_token_start_position() const noexcept
{
return token_start_position;
}
#endif
/// seekable adapter: rebuild the last read token from the input on demand
const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::true_type /*lazy*/) const
{
@@ -9451,12 +9433,6 @@ scan_number_done:
/// the last read token on error for seekable adapters (see collect_token_chars)
std::size_t token_string_start = 0;
#if JSON_DIAGNOSTIC_POSITIONS
/// start offset of the current token within the input, used to report
/// diagnostic positions (see reset())
std::size_t token_start_position = 0;
#endif
/// buffer for variable-length tokens (numbers, strings)
string_t token_buffer {};
@@ -9833,10 +9809,8 @@ class json_sax_dom_parser
case value_t::string:
{
// escape sequences make the token longer than the value it
// parses to, so the start position cannot be derived from
// the value; use the offset the lexer recorded instead
v.start_position = m_lexer_ref->get_token_start_position();
// include the length of the quotes, which is 2
v.start_position = v.end_position - v.m_data.m_value.string->size() - 2;
break;
}
@@ -10234,10 +10208,8 @@ class json_sax_dom_callback_parser
case value_t::string:
{
// escape sequences make the token longer than the value it
// parses to, so the start position cannot be derived from
// the value; use the offset the lexer recorded instead
v.start_position = m_lexer_ref->get_token_start_position();
// include the length of the quotes, which is 2
v.start_position = v.end_position - v.m_data.m_value.string->size() - 2;
break;
}
@@ -11087,6 +11059,15 @@ class binary_reader
// CBOR //
//////////
/*!
@param[in] get_char whether a new character should be retrieved from the
input (true) or whether the last read character should
be considered instead (false)
@param[in] tag_handler how CBOR tags should be treated
@return whether a valid CBOR value was passed to the SAX parser
*/
template<typename NumberType>
bool get_cbor_negative_integer()
{
@@ -11105,14 +11086,6 @@ class binary_reader
return sax->number_integer(static_cast<number_integer_t>(-1) - static_cast<number_integer_t>(number));
}
/*!
@param[in] get_char whether a new character should be retrieved from the
input (true) or whether the last read character should
be considered instead (false)
@param[in] tag_handler how CBOR tags should be treated
@return whether a valid CBOR value was passed to the SAX parser
*/
bool parse_cbor_internal(const bool get_char,
const cbor_tag_handler_t tag_handler)
{
@@ -12609,11 +12582,7 @@ class binary_reader
{
if (get_char)
{
// no get_ignore_noop() here: the byte read next must be a string
// length type specification, and a no-op ('N') is not valid in
// that position. No-ops at positions where a value may appear are
// already consumed by the callers via get_ignore_noop().
get();
get(); // TODO(niels): may we ignore N here?
}
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value")))
@@ -15968,15 +15937,19 @@ class json_pointer
// convert null values to arrays or objects before continuing
if (ptr->is_null())
{
// check if the reference token is a number
const bool nums =
std::all_of(reference_token.begin(), reference_token.end(),
[](const unsigned char x)
// check if the reference token is a valid array index, that is
// a nonempty sequence of digits without a leading '0'
// (cf. RFC 6901, Sect. 4); tokens that could never be a valid
// array index (such as "01" or "") are treated as object keys
const bool nums = !reference_token.empty()
&& (reference_token.size() == 1 || reference_token[0] != '0')
&& std::all_of(reference_token.begin(), reference_token.end(),
[](const unsigned char x)
{
return std::isdigit(x);
});
// change value to an array for numbers or "-" or to object otherwise
// change value to an array for array indices or "-" or to object otherwise
*ptr = (nums || reference_token == "-")
? detail::value_t::array
: detail::value_t::object;
@@ -18604,23 +18577,7 @@ class binary_writer
{
return true;
}
// a dimension that does not fit into std::size_t, or a product that
// overflows it, would wrap around and could match the size of
// _ArrayData_ by accident; the resulting header announces an
// element count that no reader can honor (the binary reader rejects
// it with out_of_range.408), so encode as a plain object instead
const auto dim = el.template get<std::uint64_t>();
if (!value_in_range_of<std::size_t>(dim))
{
return true;
}
const auto dim_size = static_cast<std::size_t>(dim);
if (dim_size != 0 && len > (std::numeric_limits<std::size_t>::max)() / dim_size)
{
return true;
}
len *= dim_size;
len *= static_cast<std::size_t>(el.template get<std::uint64_t>());
}
key = "_ArrayData_";
-21
View File
@@ -2730,27 +2730,6 @@ TEST_CASE("BJData")
CHECK(json::from_bjdata(json::to_bjdata(j_type), true, true) == j_type);
CHECK(json::from_bjdata(json::to_bjdata(j_size), true, true) == j_size);
}
SECTION("ndarray whose dimensions overflow stays as object")
{
// the product of the dimensions wraps around std::size_t to 0
// and so matches the size of the empty _ArrayData_; writing this
// as an ndarray would announce an element count no reader can
// honor, so it has to stay a plain object
json j_overflow = json({{"_ArrayData_", json::array()}, {"_ArraySize_", {9223372036854775808ull, 2}}, {"_ArrayType_", "uint8"}});
CHECK(json::from_bjdata(json::to_bjdata(j_overflow), true, true) == j_overflow);
// a single dimension that does not fit into std::size_t is
// rejected for the same reason (only observable where
// std::size_t is narrower than 64 bit)
json j_huge = json({{"_ArrayData_", json::array()}, {"_ArraySize_", {18446744073709551615ull}}, {"_ArrayType_", "uint8"}});
CHECK(json::from_bjdata(json::to_bjdata(j_huge), true, true) == j_huge);
// a well-formed ndarray is still encoded as one
json j_ok = json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "uint8"}});
CHECK(json::to_bjdata(j_ok) == std::vector<uint8_t>({'[', '$', 'U', '#', '[', 'i', 2, 'i', 3, ']', 1, 2, 3, 4, 5, 6}));
CHECK(json::from_bjdata(json::to_bjdata(j_ok), true, true) == j_ok);
}
}
}
-30
View File
@@ -38,36 +38,6 @@ TEST_CASE("Better diagnostics with positions")
"[json.exception.type_error.302] type must be number, but is string", json::type_error);
}
SECTION("positions of strings containing escape sequences")
{
// escape sequences make the token longer than the string it parses to,
// so the positions must not be derived from the parsed value's length
const auto check = [](const std::string & text, const std::string & token)
{
CAPTURE(text)
CAPTURE(token)
const json j = json::parse(text);
const json& v = j.at("a");
CHECK(text.substr(v.start_pos(), v.end_pos() - v.start_pos()) == token);
};
check(R"({"a":"plain"})", R"("plain")");
check(R"({"a":"tab\there"})", R"("tab\there")");
check(R"({"a":"\n\n\n\n\n\n"})", R"("\n\n\n\n\n\n")");
check(R"({"a":"\""})", R"("\"")");
check(R"({"a":"\\"})", R"("\\")");
check(R"({"a":"é"})", R"("é")");
check(R"({"a":"🌞"})", R"("🌞")");
check("{\"a\":\"\xc3\xa9\"}", "\"\xc3\xa9\""); // multi-byte UTF-8, no escapes
// a string at the root, where an escape would otherwise push the
// reported start position past the opening quote
const std::string root = R"("a\tb")";
const json j = json::parse(root);
CHECK(j.start_pos() == 0);
CHECK(j.end_pos() == root.size());
}
SECTION("JSON patch add to primitive parent (#4292)")
{
// the JSON Patch "add" target /foo/bar/baz has a string parent
+66
View File
@@ -396,6 +396,72 @@ TEST_CASE("JSON pointers")
}
}
SECTION("creating intermediate levels")
{
SECTION("tokens that are valid array indices create arrays")
{
json j;
j["/0"_json_pointer] = 1;
CHECK(j == json({1}));
json j2;
j2["/2"_json_pointer] = 1;
CHECK(j2 == json({nullptr, nullptr, 1}));
json j3;
j3["/-"_json_pointer] = 1;
CHECK(j3 == json({1}));
json j4;
j4["/foo/0/0"_json_pointer] = 1;
CHECK(j4 == json({{"foo", {{1}}}}));
}
SECTION("tokens that are no valid array indices create objects")
{
json j;
j["/one"_json_pointer] = 1;
CHECK(j == json({{"one", 1}}));
// leading '0' can never be a valid array index (RFC 6901, Sect. 4)
json j2;
j2["/01"_json_pointer] = 1;
CHECK(j2 == json({{"01", 1}}));
// the empty token is a valid object key, but no valid array index
json j3;
j3["/"_json_pointer] = 1;
CHECK(j3 == json({{"", 1}}));
}
SECTION("creating a level yields the same result as reusing it (#5357)")
{
json j;
j["/a/b/01/d"_json_pointer] = "value";
json j_init = json::object();
j_init["/a/b"_json_pointer] = json::object();
j_init["/a/b/01/d"_json_pointer] = "value";
const json expected = json::parse(R"({"a":{"b":{"01":{"d":"value"}}}})");
CHECK(j == expected);
CHECK(j_init == expected);
// unflatten uses the same key
const json flat = {{"/a/b/01/d", "value"}};
CHECK(flat.unflatten() == expected);
}
SECTION("existing arrays still reject invalid indices")
{
json j = {1, 2, 3};
CHECK_THROWS_WITH_AS(j["/01"_json_pointer],
"[json.exception.parse_error.106] parse error: array index '01' must not begin with '0'", json::parse_error&);
CHECK_THROWS_WITH_AS(j.at("/01"_json_pointer),
"[json.exception.parse_error.106] parse error: array index '01' must not begin with '0'", json::parse_error&);
}
}
SECTION("flatten")
{
json j =
-38
View File
@@ -1713,44 +1713,6 @@ TEST_CASE("UBJSON")
CHECK(json::to_ubjson(json::from_ubjson(s_L)) == s_i);
}
SECTION("no-op markers")
{
// A no-op ('N') is valid wherever a value may start; it is consumed
// by get_ignore_noop() before the value is read. It is not valid
// where a string length type specification is expected.
SECTION("accepted where a value may start")
{
// at top level, also repeated
CHECK(json::from_ubjson(std::vector<uint8_t>({'N', 'i', 1})) == json(1));
CHECK(json::from_ubjson(std::vector<uint8_t>({'N', 'N', 'N', 'i', 1})) == json(1));
// inside an array of unknown size, before and after an element
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', 'N', 'i', 1, ']'})) == json({1}));
CHECK(json::from_ubjson(std::vector<uint8_t>({'[', 'i', 1, 'N', ']'})) == json({1}));
// inside an object of unknown size: before a key, between key
// and value, and before the closing '}'
CHECK(json::from_ubjson(std::vector<uint8_t>({'{', 'N', 'U', 1, 'a', 'i', 1, '}'})) == json({{"a", 1}}));
CHECK(json::from_ubjson(std::vector<uint8_t>({'{', 'U', 1, 'a', 'N', 'i', 1, '}'})) == json({{"a", 1}}));
CHECK(json::from_ubjson(std::vector<uint8_t>({'{', 'U', 1, 'a', 'i', 1, 'N', '}'})) == json({{"a", 1}}));
}
SECTION("rejected where a length type specification is expected")
{
json _;
// after the 'S' marker of a string value
std::vector<uint8_t> const v_S = {'S', 'N', 'U', 1, 'a'};
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(v_S), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing UBJSON string: expected length type specification (U, i, I, l, L); last byte: 0x4E", json::parse_error&);
// as the key length of an object with a known size, where
// no-ops are not permitted in the first place
std::vector<uint8_t> const v_key = {'{', '#', 'i', 1, 'N', 'U', 1, 'a', 'i', 1};
CHECK_THROWS_WITH_AS(_ = json::from_ubjson(v_key), "[json.exception.parse_error.113] parse error at byte 5: syntax error while parsing UBJSON string: expected length type specification (U, i, I, l, L); last byte: 0x4E", json::parse_error&);
}
}
SECTION("number")
{
SECTION("float")
-10
View File
@@ -125,16 +125,6 @@ TEST_CASE("wide strings")
std::u32string const w = U"\"\x110000";
json _;
CHECK_THROWS_AS(_ = json::parse(w), json::parse_error&);
// a code unit above U+10FFFF must not be narrowed onto the EOF
// sentinel: 0xFFFFFFFF would otherwise end the document silently and
// let everything following it pass the strict end-of-input check
std::u32string const trailing{U'[', U'1', U']', static_cast<char32_t>(0xFFFFFFFF), U'x'};
CHECK_THROWS_WITH_AS(_ = json::parse(trailing), "[json.exception.parse_error.101] parse error at line 1, column 4: syntax error while parsing value - invalid literal; last read: '1]\xFF'; expected end of input", json::parse_error&);
CHECK(!json::accept(trailing));
// the same unit inside a string is reported as an ill-formed byte
CHECK_THROWS_WITH_AS(_ = json::parse(std::u32string{U'"', static_cast<char32_t>(0xFFFFFFFF), U'"'}), "[json.exception.parse_error.101] parse error at line 1, column 2: syntax error while parsing value - invalid string: ill-formed UTF-8 byte; last read: '\"\xFF'", json::parse_error&);
}
}
}