Compare commits

..
Author SHA1 Message Date
Niels Lohmann 31b51c95db Name the UTF-8 cut helpers after what they do to the buffer
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-17 16:43:55 +01:00
Niels Lohmann 04fbced73d Cut long tokens in parse error messages at UTF-8 character boundaries
A parse error message shows a token of more than 8192 bytes by its
first and its last 4096 bytes around "...". The cuts were at fixed byte
offsets, so they could split a UTF-8 character: for valid UTF-8 input,
what() and the last token passed to SAX parse_error() were not valid
UTF-8, and storing the message in a json value and dumping it threw
type_error.316.

Move the end of the first part back and the start of the last part
forward to a character boundary, by at most three bytes each. The
lexer always reads bytes (wide strings are converted to UTF-8 first),
and the decision depends only on bytes inside the two parts, so the
message stays the same for every kind of input. Both parts only get
shorter, so the buffers that keep the token need no more room.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-16 08:12:02 +01:00
9 changed files with 217 additions and 262 deletions

No files matched your search

-1
View File
@@ -1383,7 +1383,6 @@ Use your favorite [**package manager**](https://json.nlohmann.me/integration/pac
- [**build2**](https://json.nlohmann.me/integration/package_managers/#build2) `nlohmann-json`
- <img src="https://raw.githubusercontent.com/nlohmann/json/refs/heads/develop/docs/mkdocs/docs/images/package_managers/CPM.png" height="20">&nbsp;[**cpm.cmake**](https://json.nlohmann.me/integration/package_managers/#cpmcmake) `gh:nlohmann/json`
- <img src="https://raw.githubusercontent.com/nlohmann/json/refs/heads/develop/docs/mkdocs/docs/images/package_managers/xmake.svg" height="20">&nbsp;[**xmake**](https://json.nlohmann.me/integration/package_managers/#xmake) `nlohmann_json`
- <img src="https://raw.githubusercontent.com/nlohmann/json/refs/heads/develop/docs/mkdocs/docs/images/package_managers/nix.svg" height="20">&nbsp;[**Nix**](https://json.nlohmann.me/integration/package_managers/#nix) `nlohmann_json`
The library is part of many package managers. See the [**documentation**](https://json.nlohmann.me/integration/package_managers/) for detailed descriptions and examples.
+1 -1
View File
@@ -140,7 +140,7 @@ Exceptions have ids 1xx.
This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`.
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 bytes is shown by its first and its last 4096 bytes, separated by `...`; each part is shortened by up to three bytes so that it does not end or begin within a UTF-8 character.
!!! failure "Example message"
@@ -1,187 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
width="535"
height="535"
viewBox="0 0 501.56251 501.56249"
id="svg2"
version="1.1"
inkscape:version="1.3.2 (091e20ef0f, 2023-11-25)"
sodipodi:docname="nix-snowflake-colours.svg"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns="http://www.w3.org/2000/svg"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<defs
id="defs4">
<linearGradient
inkscape:collect="always"
id="linearGradient5562">
<stop
style="stop-color:#699ad7;stop-opacity:1"
offset="0"
id="stop5564" />
<stop
id="stop5566"
offset="0.24345198"
style="stop-color:#7eb1dd;stop-opacity:1" />
<stop
style="stop-color:#7ebae4;stop-opacity:1"
offset="1"
id="stop5568" />
</linearGradient>
<linearGradient
inkscape:collect="always"
id="linearGradient5053">
<stop
style="stop-color:#415e9a;stop-opacity:1"
offset="0"
id="stop5055" />
<stop
id="stop5057"
offset="0.23168644"
style="stop-color:#4a6baf;stop-opacity:1" />
<stop
style="stop-color:#5277c3;stop-opacity:1"
offset="1"
id="stop5059" />
</linearGradient>
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient5562"
id="linearGradient4328"
gradientUnits="userSpaceOnUse"
gradientTransform="translate(70.650339,-1055.1511)"
x1="200.59668"
y1="351.41116"
x2="290.08701"
y2="506.18814" />
<linearGradient
inkscape:collect="always"
xlink:href="#linearGradient5053"
id="linearGradient4330"
gradientUnits="userSpaceOnUse"
gradientTransform="translate(864.69589,-1491.3405)"
x1="-584.19934"
y1="782.33563"
x2="-496.29703"
y2="937.71399" />
</defs>
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="0.70904368"
inkscape:cx="99.429699"
inkscape:cy="195.33352"
inkscape:document-units="px"
inkscape:current-layer="layer3"
showgrid="false"
inkscape:window-width="1920"
inkscape:window-height="1050"
inkscape:window-x="1920"
inkscape:window-y="30"
inkscape:window-maximized="1"
inkscape:snap-global="true"
fit-margin-top="0"
fit-margin-left="0"
fit-margin-right="0"
fit-margin-bottom="0"
inkscape:showpageshadow="2"
inkscape:pagecheckerboard="0"
inkscape:deskcolor="#d1d1d1" />
<metadata
id="metadata7">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:groupmode="layer"
id="layer3"
inkscape:label="gradient-logo"
style="display:inline;opacity:1"
transform="translate(-156.41121,933.30685)">
<g
id="g2"
transform="matrix(0.99994059,0,0,0.99994059,-0.06321798,33.188377)"
style="stroke-width:1.00006">
<path
sodipodi:nodetypes="cccccccccc"
inkscape:connector-curvature="0"
id="path3336-6"
d="m 309.54892,-710.38827 122.19683,211.67512 -56.15706,0.5268 -32.6236,-56.8692 -32.85645,56.5653 -27.90237,-0.011 -14.29086,-24.6896 46.81047,-80.4901 -33.22946,-57.8257 z"
style="opacity:1;fill:url(#linearGradient4328);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3.00018;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" />
<use
height="100%"
width="100%"
transform="rotate(60,407.11155,-715.78724)"
id="use3439-6"
inkscape:transform-center-y="151.59082"
inkscape:transform-center-x="124.43045"
xlink:href="#path3336-6"
y="0"
x="0"
style="stroke-width:1.00006" />
<use
height="100%"
width="100%"
transform="rotate(-60,407.31177,-715.70016)"
id="use3445-0"
inkscape:transform-center-y="75.573958"
inkscape:transform-center-x="-168.20651"
xlink:href="#path3336-6"
y="0"
x="0"
style="stroke-width:1.00006" />
<use
height="100%"
width="100%"
transform="rotate(180,407.41868,-715.7565)"
id="use3449-5"
inkscape:transform-center-y="-139.94592"
inkscape:transform-center-x="59.669705"
xlink:href="#path3336-6"
y="0"
x="0"
style="stroke-width:1.00006" />
<path
style="color:#000000;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;opacity:1;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000000;solid-opacity:1;fill:url(#linearGradient4330);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3.00018;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate"
d="m 309.54892,-710.38827 122.19683,211.67512 -56.15706,0.5268 -32.6236,-56.8692 -32.85645,56.5653 -27.90237,-0.011 -14.29086,-24.6896 46.81047,-80.4901 -33.22946,-57.8256 z"
id="path4260-0"
inkscape:connector-curvature="0"
sodipodi:nodetypes="cccccccccc" />
<use
height="100%"
width="100%"
transform="rotate(120,407.33916,-716.08356)"
id="use4354-5"
xlink:href="#path4260-0"
y="0"
x="0"
style="display:inline;stroke-width:1.00006" />
<use
height="100%"
width="100%"
transform="rotate(-120,407.28823,-715.86995)"
id="use4362-2"
xlink:href="#path4260-0"
y="0"
x="0"
style="display:inline;stroke-width:1.00006" />
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 6.4 KiB

@@ -1,7 +0,0 @@
cmake_minimum_required(VERSION 3.15)
project(json_example)
find_package(nlohmann_json CONFIG REQUIRED)
add_executable(json_example example.cpp)
target_link_libraries(json_example PRIVATE nlohmann_json::nlohmann_json)
@@ -1,10 +0,0 @@
#include <nlohmann/json.hpp>
#include <iostream>
#include <iomanip>
using nlohmann::json;
int main()
{
std::cout << std::setw(4) << json::meta() << std::endl;
}
@@ -14,8 +14,7 @@
![Conda](../images/package_managers/conda.svg){: style="height:1em"}&nbsp;[**Conda**](#conda) `nlohmann_json` &emsp;
![MacPorts](../images/package_managers/macports.svg){: style="height:1em"}&nbsp;[**MacPorts**](#macports) `nlohmann-json`<br>
![cpm.cmake](../images/package_managers/CPM.png){: style="height:1em"}&nbsp;[**CPM.cmake**](#cpmcmake) `gh:nlohmann/json` &emsp;
![xmake](../images/package_managers/xmake.svg){: style="height:1em"}&nbsp;[**xmake**](#xmake) `nlohmann_json` &emsp;
![Nix](../images/package_managers/nix.svg){: style="height:1em"}&nbsp;[**Nix**](#nix) `nlohmann_json`
![xmake](../images/package_managers/xmake.svg){: style="height:1em"}&nbsp;[**xmake**](#xmake) `nlohmann_json`
</p>
## Running example
@@ -907,56 +906,6 @@ CPMAddPackage("gh:nlohmann/json@3.12.0")
* * *
## Nix
!!! abstract "Summary"
package: [**`nlohmann_json`**](https://search.nixos.org/packages?channel=unstable&show=nlohmann_json&query=nlohmann_json)
- :octicons-tag-24: Available versions: current version of each Nixpkgs channel (may lag behind the latest release)
- :octicons-rocket-24: The package is maintained by the Nixpkgs community and updated after releases.
- :octicons-file-24: File issues at the [Nixpkgs issue tracker](https://github.com/NixOS/nixpkgs/issues)
- :octicons-question-24: [Nix website](https://nixos.org)
If you are using [Nix](https://nixos.org) or [NixOS](https://nixos.org/manual/nixos/stable/), the library is available
as the [nlohmann_json](https://search.nixos.org/packages?channel=unstable&show=nlohmann_json&query=nlohmann_json)
package in [Nixpkgs](https://github.com/NixOS/nixpkgs). To try it in a temporary shell, execute
```shell
nix-shell -p nlohmann_json
```
On NixOS, the package can be installed system-wide by adding it to `configuration.nix`:
```nix
environment.systemPackages = [ pkgs.nlohmann_json ];
```
??? example "Example: CMake"
1. Create the following files:
```cpp title="example.cpp"
--8<-- "integration/nix/example.cpp"
```
```cmake title="CMakeLists.txt"
--8<-- "integration/nix/CMakeLists.txt"
```
2. Enter a shell that provides the package, CMake, and a C++ compiler:
```sh
nix-shell -p nlohmann_json cmake
```
3. Compile the code:
```sh
cmake -S . -B build
cmake --build build
```
## Other package managers
The library is also contained in many other package repositories; [Repology](https://repology.org/project/nlohmann-json/versions) tracks the packaged
+56 -2
View File
@@ -2279,9 +2279,14 @@ scan_number_done:
std::string result;
if (chars.size() > 2 * shown)
{
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
// the lexer reads bytes (wide characters arrive as UTF-8), so
// move both cuts to character boundaries to keep the message
// valid UTF-8; this only shortens the two parts
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(length_without_partial_utf8_at_end(chars.data(), shown));
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
append_escaped(result, chars.begin(), head_end);
result += "...";
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(partial_utf8_length_at_start(&*tail_begin)), chars.end());
}
else
{
@@ -2291,6 +2296,55 @@ scan_number_done:
}
private:
/// whether @a c is a UTF-8 continuation byte
static bool is_utf8_continuation(const char_type c) noexcept
{
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
}
/*!
@brief the length of the first @a n characters at @a data without a UTF-8
character that begins there but does not end there
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
bytes of a character can be before the cut.
*/
static std::size_t length_without_partial_utf8_at_end(const char_type* data, const std::size_t n) noexcept
{
std::size_t lead = n - 1;
while (lead > n - 4 && is_utf8_continuation(data[lead]))
{
--lead;
}
const auto c = static_cast<unsigned char>(data[lead]);
std::size_t length = 1;
if (c >= 0xF0u)
{
length = 4;
}
else if (c >= 0xE0u)
{
length = 3;
}
else if (c >= 0xC0u)
{
length = 2;
}
return lead + length > n ? lead : n;
}
/// the number of UTF-8 continuation bytes (at most 3) at the start of @a data, which
/// belong to a character that begins before it
static std::size_t partial_utf8_length_at_start(const char_type* data) noexcept
{
std::size_t skip = 0;
while (skip < 3 && is_utf8_continuation(data[skip]))
{
++skip;
}
return skip;
}
/// append the characters [first, last) to @a result, escaping control characters
template<typename Iterator>
static void append_escaped(std::string& result, Iterator first, Iterator last)
+56 -2
View File
@@ -15782,9 +15782,14 @@ scan_number_done:
std::string result;
if (chars.size() > 2 * shown)
{
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
// the lexer reads bytes (wide characters arrive as UTF-8), so
// move both cuts to character boundaries to keep the message
// valid UTF-8; this only shortens the two parts
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(length_without_partial_utf8_at_end(chars.data(), shown));
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
append_escaped(result, chars.begin(), head_end);
result += "...";
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(partial_utf8_length_at_start(&*tail_begin)), chars.end());
}
else
{
@@ -15794,6 +15799,55 @@ scan_number_done:
}
private:
/// whether @a c is a UTF-8 continuation byte
static bool is_utf8_continuation(const char_type c) noexcept
{
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
}
/*!
@brief the length of the first @a n characters at @a data without a UTF-8
character that begins there but does not end there
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
bytes of a character can be before the cut.
*/
static std::size_t length_without_partial_utf8_at_end(const char_type* data, const std::size_t n) noexcept
{
std::size_t lead = n - 1;
while (lead > n - 4 && is_utf8_continuation(data[lead]))
{
--lead;
}
const auto c = static_cast<unsigned char>(data[lead]);
std::size_t length = 1;
if (c >= 0xF0u)
{
length = 4;
}
else if (c >= 0xE0u)
{
length = 3;
}
else if (c >= 0xC0u)
{
length = 2;
}
return lead + length > n ? lead : n;
}
/// the number of UTF-8 continuation bytes (at most 3) at the start of @a data, which
/// belong to a character that begins before it
static std::size_t partial_utf8_length_at_start(const char_type* data) noexcept
{
std::size_t skip = 0;
while (skip < 3 && is_utf8_continuation(data[skip]))
{
++skip;
}
return skip;
}
/// append the characters [first, last) to @a result, escaping control characters
template<typename Iterator>
static void append_escaped(std::string& result, Iterator first, Iterator last)
+103
View File
@@ -3380,5 +3380,108 @@ TEST_CASE("parse error message of a long token")
CHECK(message(text, kind) == "no error");
}
}
SECTION("a long string of multi-byte characters")
{
// the parts of a long token are cut at character boundaries, so the
// message of valid UTF-8 input is valid UTF-8, also for wide strings
const auto last_read = [](const std::string & m) -> std::string
{
const auto pos = m.find("last read: '");
return pos == std::string::npos ? std::string() : m.substr(pos + 12, m.size() - pos - 13);
};
const auto utf8 = [](const char32_t c) -> std::string
{
std::string result;
if (c < 0x800)
{
result.push_back(static_cast<char>(0xC0 | (c >> 6)));
}
else if (c < 0x10000)
{
result.push_back(static_cast<char>(0xE0 | (c >> 12)));
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
}
else
{
result.push_back(static_cast<char>(0xF0 | (c >> 18)));
result.push_back(static_cast<char>(0x80 | ((c >> 12) & 0x3F)));
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
}
result.push_back(static_cast<char>(0x80 | (c & 0x3F)));
return result;
};
// characters of 2, 3, and 4 bytes
const std::vector<char32_t> characters = {0xE9, 0x20AC, 0x1F600};
for (const char32_t c : characters)
{
for (std::size_t offset = 0; offset < 4; ++offset)
{
CAPTURE(static_cast<std::uint32_t>(c))
CAPTURE(offset)
// an unterminated string: the token is the whole input
std::string text = "\"" + std::string(offset, 'a');
std::u32string text32 = U"\"" + std::u32string(offset, U'a');
while (text.size() < 9000)
{
text += utf8(c);
text32.push_back(c);
}
std::u16string text16;
for (const char32_t c32 : text32)
{
if (c32 < 0x10000)
{
text16.push_back(static_cast<char16_t>(c32));
}
else
{
text16.push_back(static_cast<char16_t>(0xD800 + ((c32 - 0x10000) >> 10)));
text16.push_back(static_cast<char16_t>(0xDC00 + ((c32 - 0x10000) & 0x3FF)));
}
}
const std::string shown = last_read(message(text, 0));
const auto dots = shown.find("...");
REQUIRE(dots != std::string::npos);
// the first and the last at most 4096 bytes, without parts of characters
CHECK(dots <= 4096);
CHECK(dots > 4092);
CHECK(text.compare(0, dots, shown, 0, dots) == 0);
const std::size_t tail = shown.size() - dots - 3;
CHECK(tail <= 4096);
CHECK(tail > 4092);
CHECK(text.compare(text.size() - tail, tail, shown, dots + 3, tail) == 0);
CHECK_NOTHROW(json(shown).dump());
for (int kind = 1; kind < 4; ++kind)
{
CAPTURE(kind)
CHECK(last_read(message(text, kind)) == shown);
}
json _;
std::string m32;
std::string m16;
try
{
_ = json::parse(text32);
}
catch (const json::parse_error& e)
{
m32 = e.what();
}
try
{
_ = json::parse(text16);
}
catch (const json::parse_error& e)
{
m16 = e.what();
}
CHECK(last_read(m32) == shown);
CHECK(last_read(m16) == shown);
}
}
}
}
#endif