From 6755ad6dedb2b5ce928dbd69f3df8e250ae9ff39 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 25 Oct 2026 17:32:40 +0100 Subject: [PATCH] Read ND-arrays of half floats and enforce the char limit in ND-arrays BJData N-dimensional arrays with element type 'h' (float16) failed to read with a misleading parse_error.112, because the reader's dtype name table lacked 'h'. It now maps to "half", the JData name. The writer still does not pack "half"; such an object is written as a plain object and reads back equal. BJData limits 'C' (char) to 0..127 in Draft 2 and Draft 3, and the 1-D reader enforces this, but ND-arrays with _ArrayType_ "char" wrote values up to 255 as 'C' elements and read them back without a check. The writer now only packs values in 0..127 (others fall back to a plain object that round-trips), and the reader rejects 'C' elements above 127 with parse_error.113 like the 1-D reader. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 22 +++++++ .../nlohmann/detail/output/binary_writer.hpp | 7 ++- tests/src/unit-bjdata.cpp | 61 +++++++++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index d202c8bcf..46e9905d4 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -3277,6 +3277,8 @@ class binary_reader // get_ubjson_size_value() (the type marker is known before the dimension vector // that determines size_and_type.first is read, so it is emitted first there to // match the documented _ArrayType_, _ArraySize_, _ArrayData_ key order) + // 'C' elements are read as unsigned bytes, but BJData limits a char to 0..127 + const bool nd_is_char = size_and_type.second == 'C'; if (size_and_type.second == 'C' || size_and_type.second == 'B') { size_and_type.second = 'U'; @@ -3290,6 +3292,24 @@ class binary_reader for (std::size_t i = 0; i < size_and_type.first; ++i) { + if (nd_is_char) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("char"))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(current > 127)) + { + return last_byte_error(exception_id::invalid_string_or_size, "byte after 'C' must be in range 0x00..0x7F", "char", ""); + } + if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(current)))) + { + return false; + } + continue; + } + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) { return false; @@ -4655,6 +4675,8 @@ class binary_reader return "uint8"; case 'd': return "single"; + case 'h': + return "half"; case 'i': return "int8"; case 'l': diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index d8b3e7162..8732e52b4 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -3848,7 +3848,7 @@ class binary_writer A C++11 `constexpr` function cannot contain a `switch`, so this is a plain comparison chain instead; it is only reached once per ND-array candidate object. Keep in sync with binary_reader's `bjd_type_name()`, which maps - the other way. + the other way (it also reads "half", which is never written). */ static CharType bjdata_ndarray_type_marker(const string_t& name) { @@ -4056,10 +4056,13 @@ class binary_writer switch (dtype) { case 'U': - case 'C': case 'B': ok = write_bjdata_ndarray_element(el, dry_run); break; + case 'C': + // BJData limits a char to 0..127 (the intersection of the int8 and uint8 ranges) + ok = bjdata_ndarray_value_in_range(el) && bjdata_ndarray_value_in_range(el); + break; case 'i': ok = write_bjdata_ndarray_element(el, dry_run); break; diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index db0663d52..2c929062c 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -2916,6 +2916,29 @@ TEST_CASE("BJData") CHECK(json::from_bjdata(out_single_ok) == json({{"_ArrayType_", "single"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1.5f, -1.5f}}})); } + SECTION("ndarray with _ArrayType_ \"char\" is limited to 0..127") + { + // both BJData drafts limit a char to 0..127; larger values must not + // be written as 'C' elements (the reader rejects them), so such an + // object stays a plain object that round-trips + for (const auto version : {json::bjdata_version_t::draft2, json::bjdata_version_t::draft3}) + { + json const j_ok = json({{"_ArrayType_", "char"}, {"_ArraySize_", {2, 2}}, {"_ArrayData_", {0, 97, 98, 127}}}); + const auto out_ok = json::to_bjdata(j_ok, true, true, version); + CHECK(out_ok == std::vector({'[', '$', 'C', '#', '[', '$', 'i', '#', 'i', 2, 2, 2, 0, 97, 98, 127})); + CHECK(json::from_bjdata(out_ok) == j_ok); + + for (const int bad : {128, 200, 255, 256, -1}) + { + CAPTURE(bad) + json const j_bad = json({{"_ArrayType_", "char"}, {"_ArraySize_", {2, 2}}, {"_ArrayData_", {97, 98, 99, bad}}}); + const auto out_bad = json::to_bjdata(j_bad, true, true, version); + CHECK(out_bad.at(0) == '{'); + CHECK(json::from_bjdata(out_bad) == j_bad); + } + } + } + SECTION("ndarray annotation keys are read back in the documented order") { // from_bjdata() must emit the annotation object's keys in the order @@ -3324,6 +3347,44 @@ TEST_CASE("BJData") } } + SECTION("ndarray of half-precision floats can be read") + { + // the 'h' element type is read as _ArrayType_ "half" + json _; + const std::vector half = {'[', '$', 'h', '#', '[', '$', 'U', '#', 'U', 2, 2, 2, 0x00, 0x3C, 0x00, 0x40, 0x00, 0x42, 0x00, 0x44}; + const json j_half = json::parse(R"({"_ArrayType_":"half","_ArraySize_":[2,2],"_ArrayData_":[1.0,2.0,3.0,4.0]})"); + CHECK(json::from_bjdata(half) == j_half); + + // the writer does not pack "half": it writes a plain object that reads back equal + CHECK(json::from_bjdata(json::to_bjdata(j_half)) == j_half); + CHECK(json::from_bjdata(json::to_bjdata(j_half, true, true, json::bjdata_version_t::draft3)) == j_half); + + // truncated half data + CHECK_THROWS_AS(_ = json::from_bjdata(std::vector(half.begin(), half.end() - 1)), json::parse_error&); + } + + SECTION("ndarray of char rejects values above 127") + { + json _; + const std::vector ok = {'[', '$', 'C', '#', '[', '$', 'U', '#', 'U', 2, 2, 2, 'a', 'b', 'c', 0x7F}; + CHECK(json::from_bjdata(ok) == json::parse(R"({"_ArrayType_":"char","_ArraySize_":[2,2],"_ArrayData_":[97,98,99,127]})")); + + std::vector bad = ok; + bad.back() = 0xC8; + CHECK_THROWS_AS(_ = json::from_bjdata(bad), json::parse_error&); + CHECK(json::from_bjdata(bad, true, false).is_discarded()); + + // the same restriction holds for 1-D arrays + CHECK_THROWS_AS(_ = json::from_bjdata(std::vector({'[', '$', 'C', '#', 'i', 1, 0xC8})), json::parse_error&); + + // truncated input + CHECK_THROWS_AS(_ = json::from_bjdata(std::vector(ok.begin(), ok.end() - 1)), json::parse_error&); + + // 'B' (byte) elements may use the full range + const std::vector byte_nd = {'[', '$', 'B', '#', '[', '$', 'U', '#', 'U', 2, 2, 2, 1, 2, 3, 0xC8}; + CHECK(json::from_bjdata(byte_nd) == json::parse(R"({"_ArrayType_":"byte","_ArraySize_":[2,2],"_ArrayData_":[1,2,3,200]})")); + } + SECTION("do not accept NTFZ markers in ndarray optimized type (with count)") { json _;