Move to_bson's binary subtype check into calc_bson_sizes (#5703)

to_bson() rejected a binary value's subtype above 255 (out_of_range.415)
in write_bson_binary(), which only has the binary_t, not the basic_json
value that holds it, so the exception was created with no JSON_DIAGNOSTICS
context even though the equivalent to_msgpack() check names the value's
path. The check also ran after the document size, all preceding elements,
and this element's header and length had already reached the output
adapter, so a caller-provided std::vector or std::string ended up holding
a truncated document.

calc_bson_sizes() already walks every value before anything is written,
to size embedded documents and arrays and to reject invalid keys
(out_of_range.409) up front. The subtype check now runs there instead,
in calc_bson_binary_size(), which is given the basic_json value so the
exception can use it as context. The now-redundant check in
write_bson_binary() is removed, since calc_bson_sizes() always throws
first if any binary value in the document has an oversized subtype.

Fixes #5675.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/src/unit-bson.cpp

@@ -797,7 +797,11 @@ TEST_CASE("regression test - BSON binary subtype rejects a value that doesn't fi
     CHECK(json::from_bson(json::to_bson(doc255))["b"].get_binary().subtype() == 255);
 
     CHECK_THROWS_AS(json::to_bson(json{{"b", json::binary({1, 2}, 256)}}), json::out_of_range);
-    CHECK_THROWS_WITH_AS(json::to_bson(json{{"b", json::binary({1, 2}, 300)}}), "[json.exception.out_of_range.415] subtype 300 is too large for the BSON binary subtype (max 255)", json::out_of_range);
+#if JSON_DIAGNOSTICS
+    CHECK_THROWS_WITH_AS(json::to_bson(json {{"b", json::binary({1, 2}, 300)}}), "[json.exception.out_of_range.415] (/b) subtype 300 is too large for the BSON binary subtype (max 255)", json::out_of_range);
+#else
+    CHECK_THROWS_WITH_AS(json::to_bson(json {{"b", json::binary({1, 2}, 300)}}), "[json.exception.out_of_range.415] subtype 300 is too large for the BSON binary subtype (max 255)", json::out_of_range);
+#endif
 }
 
 TEST_CASE("BSON input/output_adapters")
@@ -1805,6 +1809,21 @@ value = depth % 2 == 0 ? json{{"a", std::move(value)}, {"b", {1, "x"}}} :
         CHECK(output.empty());
     }
 
+    SECTION("a binary subtype that doesn't fit a byte is rejected before anything is written (#5675)")
+    {
+        // the offending value is nested, so this also covers that the check
+        // is not limited to a directly written value's own document
+        json const j = {{"a", {{"b", json::binary({1, 2}, 300)}}}};
+
+        std::vector<std::uint8_t> vector_output;
+        CHECK_THROWS_AS(json::to_bson(j, vector_output), json::out_of_range&);
+        CHECK(vector_output.empty());
+
+        std::string string_output;
+        CHECK_THROWS_AS(json::to_bson(j, string_output), json::out_of_range&);
+        CHECK(string_output.empty());
+    }
+
     SECTION("values nested too deeply for the call stack (#5392)")
     {
         // serializing recursed once per nesting level, and computed every

tests/src/unit-diagnostics.cpp

@@ -104,6 +104,12 @@ TEST_CASE("Regression tests for extended diagnostics")
         CHECK_THROWS_WITH_AS(j.unflatten(), "[json.exception.type_error.315] (/~1foo) values in object must be primitive", json::type_error);
     }
 
+    SECTION("Regression test for issue #5675 - to_bson: out_of_range.415 has no diagnostics context")
+    {
+        json const j = {{"a", {{"b", json::binary({1, 2}, 300)}}}};
+        CHECK_THROWS_WITH_AS(json::to_bson(j), "[json.exception.out_of_range.415] (/a/b) subtype 300 is too large for the BSON binary subtype (max 255)", json::out_of_range);
+    }
+
     SECTION("Regression test for issue #2838 - Assertion failure when inserting into arrays with JSON_DIAGNOSTICS set")
     {
         // void push_back(basic_json&& val)