Compare commits

..
Author SHA1 Message Date
Niels Lohmann 0395ad0992 Pass the complete text of a UBJSON/BJData high-precision number to SAX
For a high-precision number that is a floating-point number, the
UBJSON/BJData reader passed lexer::get_token_string() as the source
text to SAX number_float(). That helper is meant for error messages and
shows a token of more than 8192 characters only by its first and last
4096 characters around "...", so a SAX handler received a shortened
string for a long number.

Pass the number text the lexer collected while scanning the number
(lexer::get_string()) instead. It is complete and, as before, holds
only the number, without any whitespace or byte order mark around it
in the payload. Error messages still use get_token_string().

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-16 08:27:07 +01:00
12 changed files with 53 additions and 69 deletions

No files matched your search

@@ -4096,6 +4096,10 @@ class binary_reader
auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
const auto result_number = number_lexer.scan();
const auto number_string = number_lexer.get_token_string();
// the number as the lexer read it, for SAX number_float(): unlike
// get_token_string(), which is meant for error messages and shortens
// long tokens, it is complete; the next scan() replaces it
const string_t number_text = number_lexer.get_string();
const auto result_remainder = number_lexer.scan();
using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
@@ -4122,10 +4126,7 @@ class binary_reader
number_string,
out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr));
}
// number_string is a std::string, while the SAX interface takes a
// string_t; convert explicitly, as the two are only implicitly
// convertible for some string types
return sax->number_float(parsed_float, string_t(number_string.data(), number_string.size()));
return sax->number_float(parsed_float, number_text);
}
case token_type::uninitialized:
case token_type::literal_true:
+5 -4
View File
@@ -22774,6 +22774,10 @@ class binary_reader
auto number_lexer = detail::lexer<BasicJsonType, ia_type>(detail::input_adapter(number_vector), false);
const auto result_number = number_lexer.scan();
const auto number_string = number_lexer.get_token_string();
// the number as the lexer read it, for SAX number_float(): unlike
// get_token_string(), which is meant for error messages and shortens
// long tokens, it is complete; the next scan() replaces it
const string_t number_text = number_lexer.get_string();
const auto result_remainder = number_lexer.scan();
using token_type = typename detail::lexer_base<BasicJsonType>::token_type;
@@ -22800,10 +22804,7 @@ class binary_reader
number_string,
out_of_range::create(exception_id::number_overflow, concat("number overflow parsing '", number_string, '\''), nullptr));
}
// number_string is a std::string, while the SAX interface takes a
// string_t; convert explicitly, as the two are only implicitly
// convertible for some string types
return sax->number_float(parsed_float, string_t(number_string.data(), number_string.size()));
return sax->number_float(parsed_float, number_text);
}
case token_type::uninitialized:
case token_type::literal_true:
+2 -1
View File
@@ -445,7 +445,8 @@ json_test_add_test_for(src/unit-stream-bulk-scan.cpp
# with other values of JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE: 1 (the smallest),
# 1000000 (so large that the budget cannot be reached), and the largest value of
# a 64-bit integer (which saturates the limit already at the first byte, and is
# clamped to the largest std::size_t where that has 32 bits)
# clamped to the largest std::size_t where that has 32 bits). The file does not
# mention JSON_HAS_CPP_<n>, so only C++11 is built unless the standards are forced
json_test_set_test_options(test-ubjson_valueless_rate_1
COMPILE_DEFINITIONS JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE=1
)
-8
View File
@@ -6,13 +6,6 @@
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// cmake/test.cmake selects the C++ standard versions with which to build a
// unit test based on the presence of JSON_HAS_CPP_<VERSION> macros.
// When using macros that are only defined for particular versions of the standard
// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding
// version macro in a comment close by, like this:
// JSON_HAS_CPP_<VERSION> (do not remove; see note at top of file)
#include "doctest_compatibility.h"
#define JSON_TESTS_PRIVATE
@@ -22,7 +15,6 @@ using nlohmann::json;
#include <functional>
#include <limits>
#include <valarray>
// JSON_HAS_CPP_20 (do not remove; see note at top of file)
#if JSON_HAS_RANGES
#include <ranges>
#endif
+19
View File
@@ -1278,6 +1278,25 @@ TEST_CASE("BJData")
CHECK(j.dump() == "3.141592653589793");
}
SECTION("long floating-point number")
{
// the SAX parser receives the number text as it is in the
// input, also when it is longer than the part of a token
// that error messages show
const std::vector<std::size_t> lengths = {8192, 8193, 10000};
for (const auto length : lengths)
{
CAPTURE(length)
const std::string number = "1." + std::string(length - 2, '3');
std::vector<std::uint8_t> vec = {'H', 'I', static_cast<std::uint8_t>(length & 0xFF), static_cast<std::uint8_t>(length >> 8)};
vec.insert(vec.end(), number.begin(), number.end());
SaxEventLogger logger;
CHECK(json::sax_parse(vec, &logger, json::input_format_t::bjdata));
CHECK(logger.events == std::vector<std::string>({"number_float(" + number + ")"}));
CHECK(json::from_bjdata(vec).is_number_float());
}
}
SECTION("errors")
{
// error while parsing length
+2 -10
View File
@@ -6,13 +6,6 @@
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// cmake/test.cmake selects the C++ standard versions with which to build a
// unit test based on the presence of JSON_HAS_CPP_<VERSION> macros.
// When using macros that are only defined for particular versions of the standard
// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding
// version macro in a comment close by, like this:
// JSON_HAS_CPP_<VERSION> (do not remove; see note at top of file)
#include "doctest_compatibility.h"
// capture whether JSON_STRICT_NUL_HANDLING was enabled on the command line
@@ -524,7 +517,6 @@ TEST_CASE("parser class")
CHECK(json::accept(u"[1]"));
CHECK(json::parse(U"[1]") == json({1}));
CHECK(json::accept(U"[1]"));
// JSON_HAS_CPP_20 (do not remove; see note at top of file)
#if defined(__cpp_char8_t)
CHECK(json::parse(u8"[1]") == json({1}));
CHECK(json::accept(u8"[1]"));
@@ -2510,7 +2502,7 @@ TEST_CASE("text of float tokens")
std::vector<std::string> result;
std::copy_if(events.begin(), events.end(), std::back_inserter(result), [](const std::string & e)
{
return e.compare(0, 13, "number_float(") == 0; // NOLINT(modernize-use-starts-ends-with): C++11
return e.compare(0, 13, "number_float(") == 0;
});
return result;
};
@@ -3111,7 +3103,7 @@ TEST_CASE("object keys are delivered decoded and validated")
std::vector<std::string> keys;
for (const auto& e : l.events)
{
if (e.compare(0, 4, "key(") == 0) // NOLINT(modernize-use-starts-ends-with): C++11
if (e.compare(0, 4, "key(") == 0)
{
keys.push_back(e.substr(4, e.size() - 5));
}
-8
View File
@@ -6,13 +6,6 @@
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// cmake/test.cmake selects the C++ standard versions with which to build a
// unit test based on the presence of JSON_HAS_CPP_<VERSION> macros.
// When using macros that are only defined for particular versions of the standard
// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding
// version macro in a comment close by, like this:
// JSON_HAS_CPP_<VERSION> (do not remove; see note at top of file)
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
@@ -24,7 +17,6 @@
#include <type_traits>
#include <vector>
// JSON_HAS_CPP_20 (do not remove; see note at top of file)
#if JSON_HAS_THREE_WAY_COMPARISON
#include <compare>
#endif
+1 -9
View File
@@ -6,13 +6,6 @@
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// cmake/test.cmake selects the C++ standard versions with which to build a
// unit test based on the presence of JSON_HAS_CPP_<VERSION> macros.
// When using macros that are only defined for particular versions of the standard
// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding
// version macro in a comment close by, like this:
// JSON_HAS_CPP_<VERSION> (do not remove; see note at top of file)
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
@@ -279,7 +272,6 @@ struct generator
TEST_CASE("json_view string_ref")
{
// std::string_view in C++17, a stand-in with the same members before
// JSON_HAS_CPP_17 (do not remove; see note at top of file)
using nlohmann::detail::view::string_ref;
const std::string text = "abc";
const string_ref r(text);
@@ -953,7 +945,7 @@ TEST_CASE("json_view document_data::holds() (the debug check of read())")
SECTION("current edit buffer")
{
d->edits.reset(new document_data::edit_state()); // NOLINT(cppcoreguidelines-owning-memory,modernize-make-unique)
d->edits.reset(new document_data::edit_state()); // NOLINT(cppcoreguidelines-owning-memory)
CHECK(!d->holds(other.data(), other.size()));
d->edits->texts.emplace_back(new char[16]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-owning-memory)
d->edits->text_cap = 16;
-8
View File
@@ -6,13 +6,6 @@
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// cmake/test.cmake selects the C++ standard versions with which to build a
// unit test based on the presence of JSON_HAS_CPP_<VERSION> macros.
// When using macros that are only defined for particular versions of the standard
// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding
// version macro in a comment close by, like this:
// JSON_HAS_CPP_<VERSION> (do not remove; see note at top of file)
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
@@ -355,7 +348,6 @@ static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool<unsigned c
static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool<wchar_t>::value, "wchar_t is refused");
static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool<char16_t>::value, "char16_t is refused");
static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool<char32_t>::value, "char32_t is refused");
// JSON_HAS_CPP_20 (do not remove; see note at top of file)
#if defined(__cpp_char8_t)
static_assert(nlohmann::detail::recursion_depth_limit_is_char_or_bool<char8_t>::value, "char8_t is refused");
#endif
-9
View File
@@ -6,13 +6,6 @@
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// cmake/test.cmake selects the C++ standard versions with which to build a
// unit test based on the presence of JSON_HAS_CPP_<VERSION> macros.
// When using macros that are only defined for particular versions of the standard
// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding
// version macro in a comment close by, like this:
// JSON_HAS_CPP_<VERSION> (do not remove; see note at top of file)
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
@@ -30,7 +23,6 @@ using nlohmann::json;
#include <valarray>
#include <vector>
// JSON_HAS_CPP_17 (do not remove; see note at top of file)
#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1)
#if __has_include(<filesystem>)
#include <filesystem>
@@ -252,7 +244,6 @@ TEST_CASE("to_json from rvalue containers")
SECTION("std::filesystem::path rvalue is still a string")
{
CHECK(json(std::filesystem::path("a/b")) == json("a/b"));
// JSON_HAS_CPP_20 (do not remove; see note at top of file)
#ifdef __cpp_lib_char8_t
CHECK(json(std::u8string(u8"a/b")) == json("a/b"));
#endif
+19
View File
@@ -793,6 +793,25 @@ TEST_CASE("UBJSON")
CHECK(j.dump() == "3.141592653589793");
}
SECTION("long floating-point number")
{
// the SAX parser receives the number text as it is in the
// input, also when it is longer than the part of a token
// that error messages show
const std::vector<std::size_t> lengths = {8192, 8193, 10000};
for (const auto length : lengths)
{
CAPTURE(length)
const std::string number = "1." + std::string(length - 2, '3');
std::vector<std::uint8_t> vec = {'H', 'I', static_cast<std::uint8_t>(length >> 8), static_cast<std::uint8_t>(length & 0xFF)};
vec.insert(vec.end(), number.begin(), number.end());
SaxEventLogger logger;
CHECK(json::sax_parse(vec, &logger, json::input_format_t::ubjson));
CHECK(logger.events == std::vector<std::string>({"number_float(" + number + ")"}));
CHECK(json::from_ubjson(vec).is_number_float());
}
}
SECTION("errors")
{
// error while parsing length
-8
View File
@@ -6,13 +6,6 @@
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// cmake/test.cmake selects the C++ standard versions with which to build a
// unit test based on the presence of JSON_HAS_CPP_<VERSION> macros.
// When using macros that are only defined for particular versions of the standard
// (e.g., JSON_HAS_FILESYSTEM for C++17 and up), please mention the corresponding
// version macro in a comment close by, like this:
// JSON_HAS_CPP_<VERSION> (do not remove; see note at top of file)
// The budget for the elements of the UBJSON containers of a valueless type
// ('Z', 'T', 'F') of a whole input: 1048576 plus
// JSON_UBJSON_VALUELESS_ELEMENTS_PER_BYTE for every byte read (see
@@ -99,7 +92,6 @@ static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<unsigned
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<wchar_t>::value, "wchar_t is refused");
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<char16_t>::value, "char16_t is refused");
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<char32_t>::value, "char32_t is refused");
// JSON_HAS_CPP_20 (do not remove; see note at top of file)
#if defined(__cpp_char8_t)
static_assert(nlohmann::detail::valueless_elements_rate_is_char_or_bool<char8_t>::value, "char8_t is refused");
#endif