Merge branch 'develop' into claude/fix-issue-3989-db7e45

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/benchmarks/README.md

@@ -22,6 +22,15 @@ Micro-benchmarks for parsing, serialization and the binary formats, written with
 | `ViewAccept` | validate with `json_document::accept`; compare with `Accept` |
 | `ViewMaterialize` | convert a parsed `json_document` into a `json` value |
 | `ViewDump` | serialize a parsed `json_document`; compare with `Dump` |
+| `SaxParse` | parse with a SAX handler that only counts events, so no values are built |
+| `Copy`, `CopyAssign`, `CopyRing` | copy a parsed document and destroy a copy, both timed: a temporary copy, an assignment over an earlier copy, and replacing the oldest of eight live copies |
+| `CopyOnly`, `DestroyOnly` | only the copy or only the destruction, the other half untimed; a fast destructor can leave work to the next allocations, so these tell the halves apart but do not compare versions |
+| `Equal`, `Iterate` | compare a parsed document with a copy, or visit every value with range-based `for` loops |
+| `Flatten`, `FlattenOrdered`, `Unflatten` | `flatten()` for `json` and `ordered_json`, and `unflatten()` |
+| `Diff`, `Patch` | `diff()` against the document with every eighth scalar changed, and applying that patch |
+| `JsonPointerParse`, `JsonPointerToString`, `JsonPointerAt`, `JsonPointerContains` | the JSON pointers of every scalar in a document: parse them from strings, turn them back into strings, look them up with `at()` and `contains()` |
+| `GetVectorInt`, `GetVectorDouble`, `GetMap` | `get<std::vector<std::int64_t>>()`, `get<std::vector<double>>()` and `get<std::map<std::string, json>>()` on large arrays and a large flat object |
+| `ToJsonStrings` | create a `json` array from a `std::vector` of 100,000 strings, copied or moved, and destroy it again |
 
 The input files are those of [nativejson-benchmark](https://github.com/miloyip/nativejson-benchmark) (`canada`,
 `citm_catalog`, `twitter`), a large `jeopardy` file, and number-heavy files (`floats`, `signed_ints`, ...).

tests/benchmarks/src/benchmarks.cpp

@@ -9,12 +9,15 @@
 #include <benchmark/benchmark.h>
 #include <nlohmann/json.hpp>
 #include <array>
+#include <cstdint>
 #include <cstdio>
 #include <deque>
 #include <fstream>
+#include <map>
 #include <numeric>
 #include <set>
 #include <sstream>
+#include <string>
 #include <vector>
 #include <test_data.hpp>
 
@@ -710,6 +713,584 @@ static void FromCborChunkedString(benchmark::State& state, const std::size_t chu
 
 BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000);
 
+//////////////////////////////////////////////////////////////////////////////
+// SAX-parse JSON from string
+//
+// sax_parse() with a handler that does next to nothing runs the lexer and
+// parser without building values, so the gap to ParseString above is the cost
+// of building the DOM.
+//////////////////////////////////////////////////////////////////////////////
+
+static std::string read_file(const char* filename)
+{
+    std::ifstream f(filename);
+    return std::string((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
+}
+
+static json load_json(const char* filename)
+{
+    return json::parse(read_file(filename));
+}
+
+/// SAX handler that only counts the events, so that the handler costs next to nothing
+class counting_sax : public nlohmann::json_sax<json>
+{
+  public:
+    std::size_t events = 0;
+
+    bool null() override
+    {
+        return count();
+    }
+    bool boolean(bool /*val*/) override
+    {
+        return count();
+    }
+    bool number_integer(number_integer_t /*val*/) override
+    {
+        return count();
+    }
+    bool number_unsigned(number_unsigned_t /*val*/) override
+    {
+        return count();
+    }
+    bool number_float(number_float_t /*val*/, const string_t& /*s*/) override
+    {
+        return count();
+    }
+    bool string(string_t& /*val*/) override
+    {
+        return count();
+    }
+    bool binary(binary_t& /*val*/) override
+    {
+        return count();
+    }
+    bool start_object(std::size_t /*elements*/) override
+    {
+        return count();
+    }
+    bool key(string_t& /*val*/) override
+    {
+        return count();
+    }
+    bool end_object() override
+    {
+        return count();
+    }
+    bool start_array(std::size_t /*elements*/) override
+    {
+        return count();
+    }
+    bool end_array() override
+    {
+        return count();
+    }
+    bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const json::exception& /*ex*/) override
+    {
+        return false;
+    }
+
+  private:
+    bool count()
+    {
+        ++events;
+        return true;
+    }
+};
+
+static void SaxParse(benchmark::State& state, const char* filename)
+{
+    const std::string str = read_file(filename);
+
+    for (auto _ : state)
+    {
+        counting_sax sax;
+        bool result = json::sax_parse(str, &sax);
+        benchmark::DoNotOptimize(result);
+        benchmark::DoNotOptimize(sax.events);
+    }
+
+    state.SetBytesProcessed(state.iterations() * str.size());
+}
+BENCHMARK_CAPTURE(SaxParse, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(SaxParse, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(SaxParse, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+//////////////////////////////////////////////////////////////////////////////
+// work with values: copy, destroy, compare, iterate
+//
+// The parsed test files stand in for a document an application keeps around.
+// Each benchmark times one whole-document operation; setup and cleanup run
+// outside the timed section.
+//
+// A copy is always destroyed again, and how a document is freed shapes the
+// heap the next copy allocates from: a destructor that frees quickly can leave
+// work to the allocator's next calls. Copy, CopyAssign and CopyRing therefore
+// time the copy together with the destruction of a copy; CopyOnly and
+// DestroyOnly time one half each and are meant for telling the halves apart,
+// not for comparing versions.
+//////////////////////////////////////////////////////////////////////////////
+
+/// copy a document and destroy the copy again, as a temporary copy is used
+static void Copy(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+
+    for (auto _ : state)
+    {
+        json copy = j;
+        benchmark::DoNotOptimize(copy);
+    }
+}
+BENCHMARK_CAPTURE(Copy, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(Copy, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(Copy, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+/// copy-assign a document over an earlier copy of it, which is freed in the process
+static void CopyAssign(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+    json target = j;
+
+    for (auto _ : state)
+    {
+        target = j;
+        benchmark::DoNotOptimize(target);
+    }
+}
+BENCHMARK_CAPTURE(CopyAssign, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(CopyAssign, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(CopyAssign, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+/// keep eight copies alive and replace the oldest one, as a cache or an undo
+/// history does, so that the copy is not made from memory just freed by an
+/// identical document
+static void CopyRing(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+    std::vector<json> ring(8, j);
+    std::size_t oldest = 0;
+
+    for (auto _ : state)
+    {
+        ring[oldest] = j;
+        benchmark::DoNotOptimize(ring[oldest]);
+        oldest = (oldest + 1) % ring.size();
+    }
+}
+BENCHMARK_CAPTURE(CopyRing, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(CopyRing, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(CopyRing, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+/// only the copy; the copy is destroyed with the timer paused (diagnostic)
+static void CopyOnly(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+
+    for (auto _ : state)
+    {
+        auto* copy = new json(j);
+
+        state.PauseTiming();
+        delete copy;
+        state.ResumeTiming();
+    }
+}
+BENCHMARK_CAPTURE(CopyOnly, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(CopyOnly, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(CopyOnly, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+/// only the destruction; the copy is made with the timer paused (diagnostic)
+static void DestroyOnly(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+
+    for (auto _ : state)
+    {
+        state.PauseTiming();
+        auto* copy = new json(j);
+        state.ResumeTiming();
+
+        delete copy;
+    }
+}
+BENCHMARK_CAPTURE(DestroyOnly, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(DestroyOnly, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(DestroyOnly, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+static void Equal(benchmark::State& state, const char* filename)
+{
+    // two separate values, so that the comparison cannot stop at identical addresses
+    const json j = load_json(filename);
+    const json copy = j;
+
+    for (auto _ : state)
+    {
+        bool result = (j == copy);
+        benchmark::DoNotOptimize(result);
+    }
+}
+BENCHMARK_CAPTURE(Equal, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(Equal, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(Equal, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+/// visit every value with range-based for loops, as client code walks a document
+static std::size_t count_values(const json& j)
+{
+    std::size_t result = 1;
+    if (j.is_structured())
+    {
+        for (const auto& element : j)
+        {
+            result += count_values(element);
+        }
+    }
+    return result;
+}
+
+static void Iterate(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+
+    for (auto _ : state)
+    {
+        std::size_t result = count_values(j);
+        benchmark::DoNotOptimize(result);
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(count_values(j)));
+}
+BENCHMARK_CAPTURE(Iterate, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(Iterate, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(Iterate, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+//////////////////////////////////////////////////////////////////////////////
+// flatten and unflatten
+//////////////////////////////////////////////////////////////////////////////
+
+static void Flatten(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+
+    for (auto _ : state)
+    {
+        json flat = j.flatten();
+        benchmark::DoNotOptimize(flat);
+    }
+}
+BENCHMARK_CAPTURE(Flatten, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(Flatten, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(Flatten, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+// ordered_json objects look keys up linearly, which flatten() used to do for
+// every value; canada (330,000 values in one flat object) is left out, as it
+// would take minutes with such versions
+static void FlattenOrdered(benchmark::State& state, const char* filename)
+{
+    const nlohmann::ordered_json j = nlohmann::ordered_json::parse(read_file(filename));
+
+    for (auto _ : state)
+    {
+        nlohmann::ordered_json flat = j.flatten();
+        benchmark::DoNotOptimize(flat);
+    }
+}
+BENCHMARK_CAPTURE(FlattenOrdered, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(FlattenOrdered, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+static void Unflatten(benchmark::State& state, const char* filename)
+{
+    const json flat = load_json(filename).flatten();
+
+    for (auto _ : state)
+    {
+        json j = flat.unflatten();
+        benchmark::DoNotOptimize(j);
+    }
+}
+BENCHMARK_CAPTURE(Unflatten, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(Unflatten, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(Unflatten, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+//////////////////////////////////////////////////////////////////////////////
+// diff and patch
+//
+// The target is the document with every eighth scalar changed, so the patch
+// has one "replace" operation per changed value and the structure stays the
+// same.
+//////////////////////////////////////////////////////////////////////////////
+
+static void change_every_eighth_scalar(json& j, std::size_t& counter)
+{
+    if (j.is_structured())
+    {
+        for (auto& element : j)
+        {
+            change_every_eighth_scalar(element, counter);
+        }
+        return;
+    }
+
+    if (++counter % 8 != 0)
+    {
+        return;
+    }
+
+    switch (j.type())
+    {
+        case json::value_t::number_integer:
+            j = j.get<json::number_integer_t>() + 1;
+            break;
+        case json::value_t::number_unsigned:
+            j = j.get<json::number_unsigned_t>() + 1;
+            break;
+        case json::value_t::number_float:
+            j = j.get<json::number_float_t>() + 0.5;
+            break;
+        case json::value_t::string:
+            j = j.get<std::string>() + "!";
+            break;
+        case json::value_t::boolean:
+            j = !j.get<bool>();
+            break;
+        case json::value_t::null:
+        case json::value_t::object:
+        case json::value_t::array:
+        case json::value_t::binary:
+        case json::value_t::discarded:
+        default:
+            j = 0;
+            break;
+    }
+}
+
+static json changed_copy(const json& j)
+{
+    json result = j;
+    std::size_t counter = 0;
+    change_every_eighth_scalar(result, counter);
+    return result;
+}
+
+static void Diff(benchmark::State& state, const char* filename)
+{
+    const json source = load_json(filename);
+    const json target = changed_copy(source);
+
+    for (auto _ : state)
+    {
+        json patch = json::diff(source, target);
+        benchmark::DoNotOptimize(patch);
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(json::diff(source, target).size()));
+}
+BENCHMARK_CAPTURE(Diff, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(Diff, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(Diff, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+static void Patch(benchmark::State& state, const char* filename)
+{
+    const json source = load_json(filename);
+    const json patch = json::diff(source, changed_copy(source));
+
+    for (auto _ : state)
+    {
+        json result = source.patch(patch);
+        benchmark::DoNotOptimize(result);
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(patch.size()));
+}
+BENCHMARK_CAPTURE(Patch, canada,       TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json");
+BENCHMARK_CAPTURE(Patch, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(Patch, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+//////////////////////////////////////////////////////////////////////////////
+// JSON Pointer
+//
+// The pointers are those of every scalar in the document, i.e., the keys of
+// its flattened form.
+//////////////////////////////////////////////////////////////////////////////
+
+static std::vector<std::string> pointer_strings(const json& j)
+{
+    std::vector<std::string> result;
+    const json flat = j.flatten();
+    result.reserve(flat.size());
+    for (auto it = flat.begin(); it != flat.end(); ++it)
+    {
+        result.push_back(it.key());
+    }
+    return result;
+}
+
+static std::vector<json::json_pointer> pointers(const json& j)
+{
+    const std::vector<std::string> strings = pointer_strings(j);
+    std::vector<json::json_pointer> result;
+    result.reserve(strings.size());
+    for (const auto& s : strings)
+    {
+        result.emplace_back(s);
+    }
+    return result;
+}
+
+static void JsonPointerParse(benchmark::State& state, const char* filename)
+{
+    const std::vector<std::string> strings = pointer_strings(load_json(filename));
+
+    for (auto _ : state)
+    {
+        for (const auto& s : strings)
+        {
+            json::json_pointer ptr(s);
+            benchmark::DoNotOptimize(ptr);
+        }
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(strings.size()));
+}
+BENCHMARK_CAPTURE(JsonPointerParse, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(JsonPointerParse, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+static void JsonPointerToString(benchmark::State& state, const char* filename)
+{
+    const std::vector<json::json_pointer> ptrs = pointers(load_json(filename));
+
+    for (auto _ : state)
+    {
+        for (const auto& ptr : ptrs)
+        {
+            std::string s = ptr.to_string();
+            benchmark::DoNotOptimize(s);
+        }
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
+}
+BENCHMARK_CAPTURE(JsonPointerToString, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(JsonPointerToString, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+static void JsonPointerAt(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+    const std::vector<json::json_pointer> ptrs = pointers(j);
+
+    for (auto _ : state)
+    {
+        for (const auto& ptr : ptrs)
+        {
+            const json& value = j.at(ptr);
+            benchmark::DoNotOptimize(&value);
+        }
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
+}
+BENCHMARK_CAPTURE(JsonPointerAt, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(JsonPointerAt, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+static void JsonPointerContains(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+    const std::vector<json::json_pointer> ptrs = pointers(j);
+
+    for (auto _ : state)
+    {
+        for (const auto& ptr : ptrs)
+        {
+            bool result = j.contains(ptr);
+            benchmark::DoNotOptimize(result);
+        }
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(ptrs.size()));
+}
+BENCHMARK_CAPTURE(JsonPointerContains, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_CAPTURE(JsonPointerContains, twitter,      TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json");
+
+//////////////////////////////////////////////////////////////////////////////
+// conversions from and to C++ containers
+//////////////////////////////////////////////////////////////////////////////
+
+static void GetVectorInt(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+
+    for (auto _ : state)
+    {
+        auto v = j.get<std::vector<std::int64_t>>();
+        benchmark::DoNotOptimize(v);
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
+}
+BENCHMARK_CAPTURE(GetVectorInt, signed_ints, TEST_DATA_DIRECTORY "/regression/signed_ints.json");
+
+static void GetVectorDouble(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename);
+
+    for (auto _ : state)
+    {
+        auto v = j.get<std::vector<double>>();
+        benchmark::DoNotOptimize(v);
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
+}
+BENCHMARK_CAPTURE(GetVectorDouble, floats, TEST_DATA_DIRECTORY "/regression/floats.json");
+
+// a large flat object: the flattened document, one member per scalar
+static void GetMap(benchmark::State& state, const char* filename)
+{
+    const json j = load_json(filename).flatten();
+
+    for (auto _ : state)
+    {
+        auto m = j.get<std::map<std::string, json>>();
+        benchmark::DoNotOptimize(m);
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(j.size()));
+}
+BENCHMARK_CAPTURE(GetMap, citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json");
+
+/// 100,000 strings, each too long for the small string optimization
+static std::vector<std::string> make_strings()
+{
+    std::vector<std::string> result;
+    result.reserve(100000);
+    for (std::size_t i = 0; i < 100000; ++i)
+    {
+        result.push_back("a string that does not fit in place: " + std::to_string(i));
+    }
+    return result;
+}
+
+static void ToJsonStrings(benchmark::State& state, const bool move)
+{
+    const std::vector<std::string> strings = make_strings();
+
+    for (auto _ : state)
+    {
+        state.PauseTiming();
+        std::vector<std::string> input = strings;
+        state.ResumeTiming();
+
+        // destroyed inside the timed section, like the copies above
+        auto* j = move ? new json(std::move(input)) : new json(input);
+        delete j;
+    }
+
+    state.SetItemsProcessed(state.iterations() * static_cast<std::int64_t>(strings.size()));
+}
+BENCHMARK_CAPTURE(ToJsonStrings, copy, false);
+BENCHMARK_CAPTURE(ToJsonStrings, move, true);
+
 // conversion of a JSON array into standard containers
 template<typename Container>
 static void GetArray(benchmark::State& state)

tests/src/unit-class_lexer.cpp

@@ -15,9 +15,12 @@ using nlohmann::json;
 #include <array> // array
 #include <cstdint> // uint32_t, uint64_t
 #include <cstdio> // snprintf
+#include <cmath> // isfinite
 #include <cstdlib> // strtod
 #include <cstring> // memcpy
 #include <limits> // numeric_limits
+#include <initializer_list> // initializer_list
+#include <list> // list
 #include <map> // map
 #include <random> // mt19937
 #include <sstream> // stringstream
@@ -332,6 +335,112 @@ TEST_CASE("lexer number fast path")
         }
     }
 
+#if !defined(JSON_NOEXCEPTION)
+    // this section reports parse errors, which abort when exceptions are off
+    SECTION("digits accumulated while scanning")
+    {
+        // The fast path accumulates the digits while it validates them, eight
+        // at a time where possible, and converts tokens with at most 19
+        // digits (floats) or 18 digits (integers) from that value. Every digit
+        // count from 1 to 25, in the integer and in the fraction part, with
+        // and without sign and exponent, must give the byte path's result. A
+        // std::list has no contiguous storage, so parsing it takes the byte
+        // path.
+        // type and value, or the error (values beyond the range of double
+        // are rejected with out_of_range.406 on both paths)
+        const auto outcome = [](const std::string & doc, bool byte_path) -> std::string
+        {
+            JSON_TRY
+            {
+                const std::list<char> chars(doc.begin(), doc.end());
+                const json j = byte_path ? json::parse(chars.begin(), chars.end()) : json::parse(doc);
+                return std::string(j[0].type_name()) + '|' + j.dump();
+            }
+            JSON_CATCH (const json::exception & e)
+            {
+                return e.what();
+            }
+        };
+
+        // the concatenation of @a parts
+        const auto join = [](std::initializer_list<std::string> parts)
+        {
+            std::string result;
+            for (const auto& part : parts)
+            {
+                result += part;
+            }
+            return result;
+        };
+
+        std::vector<std::string> numbers;
+        for (std::size_t digits = 1; digits <= 25; ++digits)
+        {
+            std::string run;
+            for (std::size_t i = 0; i < digits; ++i)
+            {
+                run += static_cast<char>('1' + static_cast<char>((i * 7) % 9));
+            }
+            const std::string nines(digits, '9');
+            for (const char* sign_text :
+                    {"", "-"
+                    })
+            {
+                const std::string sign = sign_text;
+                numbers.push_back(join({sign, run}));
+                numbers.push_back(join({sign, nines}));
+                numbers.push_back(join({sign, "1", std::string(digits - 1, '0')}));
+                numbers.push_back(join({sign, "0.", run}));
+                numbers.push_back(join({sign, "0.", std::string(digits, '0'), "1"}));
+                numbers.push_back(join({sign, run, ".", run}));
+                numbers.push_back(join({sign, run, "e-", std::to_string(digits)}));
+                numbers.push_back(join({sign, "1.", run, "E+", std::to_string(digits * 13)}));
+            }
+        }
+        // exponents beyond every range, and zeros
+        numbers.insert(numbers.end(), {"1e400", "1e-400", "-1e99999999999999999999", "1e-99999999999999999999",
+                                       "0e99999999999999999999", "0.0e-5", "-0.000", "123456789e-0000000000000000000000330"
+                                      });
+
+        for (const auto& n : numbers)
+        {
+            CAPTURE(n)
+            const std::string doc = "[" + n + "]";
+            const std::string a = outcome(doc, false);
+            CHECK(a == outcome(doc, true));
+            const double expected = std::strtod(n.c_str(), nullptr);
+            if (a.compare(0, 6, "number") == 0 && std::isfinite(expected) && n.find_first_of(".eE") != std::string::npos)
+            {
+                CHECK(json::parse(doc)[0].get<double>() == expected);
+            }
+        }
+    }
+#endif
+
+    SECTION("integers that do not fit a narrower type become floats")
+    {
+        // up to 18 digits, the fast path stores the accumulated value without
+        // an overflow check; a narrower number_integer_t/number_unsigned_t
+        // must still see the value rejected and converted as a float
+        using json32 = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int32_t, std::uint32_t, float>;
+        for (const char* n :
+                {"4294967295", "4294967296", "2147483647", "-2147483648", "-2147483649", "99999999999", "-99999999999"
+                })
+        {
+            CAPTURE(n)
+            const std::string doc = std::string("[") + n + "]";
+            const std::list<char> chars(doc.begin(), doc.end());
+            const json32 a = json32::parse(doc);
+            const json32 b = json32::parse(chars.begin(), chars.end());
+            CHECK(a[0].type() == b[0].type());
+            CHECK(a == b);
+        }
+        CHECK(json32::parse("[4294967295]")[0].is_number_unsigned());
+        CHECK(json32::parse("[4294967296]")[0].is_number_float());
+        CHECK(json32::parse("[-2147483648]")[0].is_number_integer());
+        CHECK(json32::parse("[-2147483649]")[0].is_number_float());
+    }
+
     SECTION("token type classification")
     {
         CHECK((scan_string("0") == json::lexer::token_type::value_unsigned));

tests/src/unit-class_parser.cpp

@@ -30,7 +30,9 @@ using nlohmann::json;
 #include <valarray>
 #include <algorithm>
 #include <cstdio>
+#include <cstdint>
 #include <fstream>
+#include <iterator>
 #include <list>
 #include <sstream>
 #include <string>
@@ -2450,6 +2452,79 @@ std::string parse_error_message_range(IteratorType first, IteratorType last)
 }
 } // namespace
 
+TEST_CASE("text of float tokens")
+{
+    // For contiguous input, the lexer copies the text of a float token into
+    // its buffer only when a SAX handler asks for it (the library's own
+    // handlers do not). A user handler must still see the exact text, for
+    // every input adapter and in every position of a document.
+    const std::string doc = R"([1.5, -0.0, 0.1, 1e5, -2.5E-4, 12345678901234567890.5, 0.12345678901234567890123,
+                              {"a": 3.25, "b": [6.02e23, 1]}, "x", true, 7e-1])";
+    const std::vector<std::string> expected_floats =
+    {
+        "number_float(1.5)", "number_float(-0.0)", "number_float(0.1)", "number_float(1e5)", "number_float(-2.5E-4)",
+        "number_float(12345678901234567890.5)", "number_float(0.12345678901234567890123)", "number_float(3.25)",
+        "number_float(6.02e23)", "number_float(7e-1)"
+    };
+
+    const auto floats = [](const std::vector<std::string>& events)
+    {
+        std::vector<std::string> result;
+        std::copy_if(events.begin(), events.end(), std::back_inserter(result), [](const std::string & e)
+        {
+            return e.compare(0, 13, "number_float(") == 0;
+        });
+        return result;
+    };
+
+    SECTION("contiguous input")
+    {
+        SaxEventLogger l;
+        CHECK(json::sax_parse(doc, &l));
+        CHECK(floats(l.events) == expected_floats);
+    }
+
+    SECTION("byte input")
+    {
+        const std::vector<std::uint8_t> bytes(doc.begin(), doc.end());
+        SaxEventLogger l;
+        CHECK(json::sax_parse(bytes, &l));
+        CHECK(floats(l.events) == expected_floats);
+    }
+
+    SECTION("stream input")
+    {
+        std::istringstream is(doc);
+        SaxEventLogger l;
+        CHECK(json::sax_parse(is, &l));
+        CHECK(floats(l.events) == expected_floats);
+    }
+
+    SECTION("non-contiguous iterators")
+    {
+        const std::list<char> chars(doc.begin(), doc.end());
+        SaxEventLogger l;
+        CHECK(json::sax_parse(chars.begin(), chars.end(), &l));
+        CHECK(floats(l.events) == expected_floats);
+    }
+
+    SECTION("the library's own handlers do not need the text")
+    {
+        using lexer_adapter = nlohmann::detail::iterator_input_adapter<const char*>;
+        CHECK(nlohmann::detail::sax_ignores_number_text<nlohmann::detail::json_sax_dom_parser<json, lexer_adapter>>::value);
+        CHECK(nlohmann::detail::sax_ignores_number_text<nlohmann::detail::json_sax_dom_callback_parser<json, lexer_adapter>>::value);
+        CHECK(nlohmann::detail::sax_ignores_number_text<nlohmann::detail::json_sax_acceptor<json>>::value);
+        CHECK_FALSE(nlohmann::detail::sax_ignores_number_text<SaxEventLogger>::value);
+
+        // and the values are the same as when the text is read
+        const json j = json::parse(doc);
+        CHECK(j[0] == 1.5);
+        CHECK(j[5].get<double>() == 12345678901234567890.5);
+        CHECK(j[7]["b"][0] == 6.02e23);
+        CHECK(json::accept(doc));
+    }
+}
+
 TEST_CASE("last-read diagnostics are identical across input adapters")
 {
     // The lexer reconstructs the "last read" token lazily for seekable adapters