From 65d5e6604234043b16a7567373360c236ae2d3f0 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 2 Sep 2026 23:27:52 +0200 Subject: [PATCH] Do not search a container for the value the callback rejected When a parser callback rejects a value, the placeholder stored for it has to be removed from its parent again. remove_discarded_value() found it by scanning the parent from the beginning, so filtering a container cost one scan per rejected member - quadratic in the number of members of a single container. A rejected value can only ever be the one most recently added to its parent: the last element of an array, or the placeholder key() stored under the current key in an object. Record that key alongside the existing key_keep_stack, and for a container record it again alongside ref_stack so end_object()/end_array() can find it in the parent. Removal is then O(1) for an array and O(log n) for an object, and finding nothing there means nothing was stored, so there is nothing to remove. The key for a container is read before handle_value() may consume it, so it is also correct when the callback rejects the container at its start event and it never reaches its parent at all. Discarding half the members of one object, before -> after: members value rejected container rejected at start 16 000 392 ms -> 3.7 ms 803 ms -> 8.2 ms 64 000 6238 ms -> 14.6 ms 12651 ms -> 32.2 ms 128 000 25339 ms -> 30.6 ms Results are unchanged: 48 000 randomized documents parsed under 12 different filtering callbacks - covering duplicate keys, empty keys, rejected keys and containers rejected at both their start and end events - produce byte identical output before and after. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/json_sax.hpp | 87 +++++++++++++++++++--- single_include/nlohmann/json.hpp | 87 +++++++++++++++++++--- tests/src/unit-class_parser.cpp | 52 +++++++++++++ 3 files changed, 208 insertions(+), 18 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 8a98ee728..684fac047 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -548,6 +548,11 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); keep_stack.push_back(keep); + // the key this object will be stored under, read before handle_value() + // may consume it; kept in lockstep with ref_stack so end_object() can + // find the object in its parent again + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::object, true); ref_stack.push_back(val.second); @@ -581,6 +586,9 @@ class json_sax_dom_callback_parser // check callback for the key const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); key_keep_stack.push_back(keep); + // remember the key so a rejected value can be erased without searching + // the object for it (kept in lockstep with key_keep_stack) + key_stack.push_back(val); // add discarded value at the given key and store the reference for later if (keep && ref_stack.back()) @@ -622,13 +630,16 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) { // remove discarded value - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } return true; @@ -639,6 +650,9 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); keep_stack.push_back(keep); + // see start_object() + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::array, true); ref_stack.push_back(val.second); @@ -701,8 +715,11 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); // remove discarded value if (!ref_stack.empty() && ref_stack.back()) @@ -716,7 +733,7 @@ class json_sax_dom_callback_parser // the array is either still stored under its key or was never // stored, leaving the placeholder key() wrote; both show up as // a discarded member of the parent object - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } } @@ -809,15 +826,56 @@ class json_sax_dom_callback_parser } #endif - /// remove the discarded value the callback rejected from its parent - static void remove_discarded_value(BasicJsonType& parent) + /*! + @brief the key the value now being handled will be stored under + + Empty unless the enclosing container is an object, in which case it is the + key of the pending key() event. Read before handle_value() consumes that + key, so it is also correct when the value never reaches its parent. + */ + string_t current_key() const { - for (auto it = parent.begin(); it != parent.end(); ++it) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() + && !key_stack.empty()) { - if (it->is_discarded()) + return key_stack.back(); + } + return string_t{}; + } + + /*! + @brief remove the discarded value the callback rejected from its parent + + A rejected value can only ever be the one most recently added to @a parent: + the last element of an array, or the placeholder key() stored under @a key + in an object. Looking there directly makes this O(1) resp. O(log n), where + searching @a parent for it made a filtering parse quadratic in the number of + members of a single container. + + Finding no discarded value there means none was stored in the first place - + the callback rejected the value before it reached its parent - so there is + nothing to remove. + + @param[in,out] parent the container to remove the rejected value from + @param[in] key the key the value was stored under; unused for arrays + */ + static void remove_discarded_value(BasicJsonType& parent, const string_t& key) + { + if (parent.is_array()) + { + auto& array = *parent.m_data.m_value.array; + if (!array.empty() && array.back().is_discarded()) { - parent.erase(it); - break; + array.pop_back(); + } + } + else if (parent.is_object()) + { + auto& object = *parent.m_data.m_value.object; + const auto it = object.find(key); + if (it != object.end() && it->second.is_discarded()) + { + object.erase(it); } } } @@ -867,11 +925,14 @@ class json_sax_dom_callback_parser if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool placeholder_stored = key_keep_stack.back(); key_keep_stack.pop_back(); + const string_t key = std::move(key_stack.back()); + key_stack.pop_back(); if (placeholder_stored) { - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), key); } } return {false, nullptr}; @@ -904,8 +965,10 @@ class json_sax_dom_callback_parser JSON_ASSERT(ref_stack.back()->is_object()); // check if we should store an element for the current key JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool store_element = key_keep_stack.back(); key_keep_stack.pop_back(); + key_stack.pop_back(); if (!store_element) { @@ -925,6 +988,12 @@ class json_sax_dom_callback_parser std::vector keep_stack {}; // NOLINT(readability-redundant-member-init) /// stack to manage which object keys to keep std::vector key_keep_stack {}; // NOLINT(readability-redundant-member-init) + /// the keys key() stored a placeholder for, in lockstep with key_keep_stack + std::vector key_stack {}; // NOLINT(readability-redundant-member-init) + /// for each open container, the key it is stored under in its parent + /// object, in lockstep with ref_stack; unused where the parent is not an + /// object + std::vector container_key_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; /// whether a syntax error occurred diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c15f94491..b8da1e526 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11065,6 +11065,11 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); keep_stack.push_back(keep); + // the key this object will be stored under, read before handle_value() + // may consume it; kept in lockstep with ref_stack so end_object() can + // find the object in its parent again + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::object, true); ref_stack.push_back(val.second); @@ -11098,6 +11103,9 @@ class json_sax_dom_callback_parser // check callback for the key const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); key_keep_stack.push_back(keep); + // remember the key so a rejected value can be erased without searching + // the object for it (kept in lockstep with key_keep_stack) + key_stack.push_back(val); // add discarded value at the given key and store the reference for later if (keep && ref_stack.back()) @@ -11139,13 +11147,16 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) { // remove discarded value - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } return true; @@ -11156,6 +11167,9 @@ class json_sax_dom_callback_parser const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); keep_stack.push_back(keep); + // see start_object() + container_key_stack.push_back(current_key()); + auto val = handle_value(BasicJsonType::value_t::array, true); ref_stack.push_back(val.second); @@ -11218,8 +11232,11 @@ class json_sax_dom_callback_parser JSON_ASSERT(!ref_stack.empty()); JSON_ASSERT(!keep_stack.empty()); + JSON_ASSERT(!container_key_stack.empty()); ref_stack.pop_back(); keep_stack.pop_back(); + const string_t object_key = std::move(container_key_stack.back()); + container_key_stack.pop_back(); // remove discarded value if (!ref_stack.empty() && ref_stack.back()) @@ -11233,7 +11250,7 @@ class json_sax_dom_callback_parser // the array is either still stored under its key or was never // stored, leaving the placeholder key() wrote; both show up as // a discarded member of the parent object - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), object_key); } } @@ -11326,15 +11343,56 @@ class json_sax_dom_callback_parser } #endif - /// remove the discarded value the callback rejected from its parent - static void remove_discarded_value(BasicJsonType& parent) + /*! + @brief the key the value now being handled will be stored under + + Empty unless the enclosing container is an object, in which case it is the + key of the pending key() event. Read before handle_value() consumes that + key, so it is also correct when the value never reaches its parent. + */ + string_t current_key() const { - for (auto it = parent.begin(); it != parent.end(); ++it) + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object() + && !key_stack.empty()) { - if (it->is_discarded()) + return key_stack.back(); + } + return string_t{}; + } + + /*! + @brief remove the discarded value the callback rejected from its parent + + A rejected value can only ever be the one most recently added to @a parent: + the last element of an array, or the placeholder key() stored under @a key + in an object. Looking there directly makes this O(1) resp. O(log n), where + searching @a parent for it made a filtering parse quadratic in the number of + members of a single container. + + Finding no discarded value there means none was stored in the first place - + the callback rejected the value before it reached its parent - so there is + nothing to remove. + + @param[in,out] parent the container to remove the rejected value from + @param[in] key the key the value was stored under; unused for arrays + */ + static void remove_discarded_value(BasicJsonType& parent, const string_t& key) + { + if (parent.is_array()) + { + auto& array = *parent.m_data.m_value.array; + if (!array.empty() && array.back().is_discarded()) { - parent.erase(it); - break; + array.pop_back(); + } + } + else if (parent.is_object()) + { + auto& object = *parent.m_data.m_value.object; + const auto it = object.find(key); + if (it != object.end() && it->second.is_discarded()) + { + object.erase(it); } } } @@ -11384,11 +11442,14 @@ class json_sax_dom_callback_parser if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object()) { JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool placeholder_stored = key_keep_stack.back(); key_keep_stack.pop_back(); + const string_t key = std::move(key_stack.back()); + key_stack.pop_back(); if (placeholder_stored) { - remove_discarded_value(*ref_stack.back()); + remove_discarded_value(*ref_stack.back(), key); } } return {false, nullptr}; @@ -11421,8 +11482,10 @@ class json_sax_dom_callback_parser JSON_ASSERT(ref_stack.back()->is_object()); // check if we should store an element for the current key JSON_ASSERT(!key_keep_stack.empty()); + JSON_ASSERT(!key_stack.empty()); const bool store_element = key_keep_stack.back(); key_keep_stack.pop_back(); + key_stack.pop_back(); if (!store_element) { @@ -11442,6 +11505,12 @@ class json_sax_dom_callback_parser std::vector keep_stack {}; // NOLINT(readability-redundant-member-init) /// stack to manage which object keys to keep std::vector key_keep_stack {}; // NOLINT(readability-redundant-member-init) + /// the keys key() stored a placeholder for, in lockstep with key_keep_stack + std::vector key_stack {}; // NOLINT(readability-redundant-member-init) + /// for each open container, the key it is stored under in its parent + /// object, in lockstep with ref_stack; unused where the parent is not an + /// object + std::vector container_key_stack {}; // NOLINT(readability-redundant-member-init) /// helper to hold the reference for the next object element BasicJsonType* object_element = nullptr; /// whether a syntax error occurred diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 8b3ea660e..d223868af 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -1564,6 +1564,58 @@ TEST_CASE("parser class") CHECK (j_filtered2 == json({{"foo", {1, 2}}})); } + SECTION("filter many members of one container") + { + // Rejecting a value makes the parser remove the placeholder its key + // event stored. Locating that placeholder used to be a scan of the + // whole parent, which made filtering a large container quadratic: + // 128k members took ~25 s. These cases keep many members alive + // while discarding many others, so the removal cost is the whole + // point; they run in milliseconds when the placeholder is erased + // directly. + constexpr int count = 20000; + + std::string s = "{"; + for (int i = 0; i < count; ++i) + { + // "a" is kept, "z" is discarded + s += "\"a" + std::to_string(i) + "\":" + std::to_string(i) + ","; + s += "\"z" + std::to_string(i) + "\":-1,"; + } + s.back() = '}'; + + const json j_values = json::parse(s, [](int /*unused*/, json::parse_event_t e, const json & parsed) noexcept + { + return !(e == json::parse_event_t::value && parsed == json(-1)); + }); + + CHECK(j_values.size() == count); + CHECK(j_values.at("a0") == json(0)); + CHECK(j_values.at("a" + std::to_string(count - 1)) == json(count - 1)); + CHECK_FALSE(j_values.contains("z0")); + CHECK_FALSE(j_values.contains("z" + std::to_string(count - 1))); + + // the same, but discarding whole containers rather than values, + // which takes the end_object()/end_array() removal path + std::string s_nested = "{"; + for (int i = 0; i < count; ++i) + { + s_nested += "\"a" + std::to_string(i) + "\":" + std::to_string(i) + ","; + s_nested += "\"z" + std::to_string(i) + "\":[1,2],"; + } + s_nested.back() = '}'; + + const json j_arrays = json::parse(s_nested, [](int /*unused*/, json::parse_event_t e, const json& /*unused*/) noexcept + { + return e != json::parse_event_t::array_end; + }); + + CHECK(j_arrays.size() == count); + CHECK(j_arrays.at("a0") == json(0)); + CHECK_FALSE(j_arrays.contains("z0")); + CHECK_FALSE(j_arrays.contains("z" + std::to_string(count - 1))); + } + SECTION("filter specific events") { SECTION("first closing event")