Support UBJSON-derived Binary JData (BJData) format (#3336)

* support UBJSON-derived Binary JData (BJData) format

* fix Codacy warning

* partially fix VS compilation errors

* fix additional VS errors

* fix more VS compilation errors

* fix additional warnings and errors for clang and msvc

* add more tests to cover the new bjdata types

* add tests for optimized ndarray, improve coverage, fix clang/gcc warnings

* gcc warn useless conversion but msvc gives an error

* fix ci_test errors

* complete test coverage, fix ci_test errors

* add half precision error test

* fix No newline at end of file error by clang

* simplify endian condition, format unit-bjdata

* remove broken test due to alloc limit

* full coverage, I hope

* move bjdata new markers from default to the same level as ubjson markers

* fix ci errors, add tests for new bjdata switch structure

* make is_bjdata const after using initializer list

* remove the unwanted assert

* move is_bjdata to an optional param to write_ubjson

* pass use_bjdata via output adapter

* revert order to avoid msvc 2015 unreferenced formal param error

* update BJData Spect V1 Draft-2 URL after spec release

* amalgamate code

* code polishing following @gregmarr's feedback

* make use_bjdata a non-default parameter

* fix ci error, remove unwanted param comment

* encode and decode bjdata ndarray in jdata annotations, enable roundtrip tests

* partially fix ci errors, add tests to improve coverage

* polish patch to remove ci errors

* fix a ndarray dim vector condition

* fix clang tidy error

* add sax test cases for ndarray

* add additional sax event tests

* adjust sax event numbering

* fix sax tests

* ndarray can only be used with array containers, discard if used in object

* complete test coverage

* disable [{SHTFNZ in optimized type due to security risks in #2793 and hampered readability

* fix ci error

* move OutputIsLittleEndian from tparam to param to replace use_bjdata

* fix ci clang gcc error

* fix ci static analysis error

* update json_test_data to 3.1.0, enable file-based bjdata unit tests

* fix stack overflow error on msvc 2019 and 2022

* use https link, update sax_parse_error after rebase

* make input_format const and use initializer

* return bool for write_bjdata_ndarray

* test write_bjdata_ndarray return value as boolean

* fix ci error

test/CMakeLists.txt

@@ -72,7 +72,7 @@ endif()
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
     # avoid stack overflow, see https://github.com/nlohmann/json/issues/2955
-    json_test_set_test_options("test-cbor;test-msgpack;test-ubjson" LINK_OPTIONS /STACK:4000000)
+    json_test_set_test_options("test-cbor;test-msgpack;test-ubjson;test-bjdata" LINK_OPTIONS /STACK:4000000)
 endif()
 
 # disable exceptions for test-disabled_exceptions

test/Makefile

@@ -10,7 +10,7 @@ CXXFLAGS += -std=c++11
 CPPFLAGS += -I ../single_include
 
 FUZZER_ENGINE = src/fuzzer-driver_afl.cpp
-FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer
+FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer parse_bjdata_fuzzer
 fuzzers: $(FUZZERS)
 
 parse_afl_fuzzer:
@@ -27,3 +27,6 @@ parse_msgpack_fuzzer:
 
 parse_ubjson_fuzzer:
 	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_ubjson.cpp -o $@
+
+parse_bjdata_fuzzer:
+	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_bjdata.cpp -o $@

test/src/fuzzer-parse_bjdata.cpp

@@ -0,0 +1,84 @@
+/*
+    __ _____ _____ _____
+ __|  |   __|     |   | |  JSON for Modern C++ (fuzz test support)
+|  |  |__   |  |  | | | |  version 3.10.5
+|_____|_____|_____|_|___|  https://github.com/nlohmann/json
+
+This file implements a parser test suitable for fuzz testing. Given a byte
+array data, it performs the following steps:
+
+- j1 = from_bjdata(data)
+- vec = to_bjdata(j1)
+- j2 = from_bjdata(vec)
+- assert(j1 == j2)
+- vec2 = to_bjdata(j1, use_size = true, use_type = false)
+- j3 = from_bjdata(vec2)
+- assert(j1 == j3)
+- vec3 = to_bjdata(j1, use_size = true, use_type = true)
+- j4 = from_bjdata(vec3)
+- assert(j1 == j4)
+
+The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
+drivers.
+
+Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+*/
+
+#include <iostream>
+#include <sstream>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+// see http://llvm.org/docs/LibFuzzer.html
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+{
+    try
+    {
+        // step 1: parse input
+        std::vector<uint8_t> vec1(data, data + size);
+        json j1 = json::from_bjdata(vec1);
+
+        try
+        {
+            // step 2.1: round trip without adding size annotations to container types
+            std::vector<uint8_t> vec2 = json::to_bjdata(j1, false, false);
+
+            // step 2.2: round trip with adding size annotations but without adding type annonations to container types
+            std::vector<uint8_t> vec3 = json::to_bjdata(j1, true, false);
+
+            // step 2.3: round trip with adding size as well as type annotations to container types
+            std::vector<uint8_t> vec4 = json::to_bjdata(j1, true, true);
+
+            // parse serialization
+            json j2 = json::from_bjdata(vec2);
+            json j3 = json::from_bjdata(vec3);
+            json j4 = json::from_bjdata(vec4);
+
+            // serializations must match
+            assert(json::to_bjdata(j2, false, false) == vec2);
+            assert(json::to_bjdata(j3, true, false) == vec3);
+            assert(json::to_bjdata(j4, true, true) == vec4);
+        }
+        catch (const json::parse_error&)
+        {
+            // parsing a BJData serialization must not fail
+            assert(false);
+        }
+    }
+    catch (const json::parse_error&)
+    {
+        // parse errors are ok, because input may be random bytes
+    }
+    catch (const json::type_error&)
+    {
+        // type errors can occur during parsing, too
+    }
+    catch (const json::out_of_range&)
+    {
+        // out of range errors may happen if provided sizes are excessive
+    }
+
+    // return 0 - non-zero return values are reserved for future use
+    return 0;
+}