remove discarded array from parent object in end_array (#5342)

* remove discarded array from parent object in end_array

Signed-off-by: Angadi Yashaswini <angadi@digiscrypt.com>

* remove discarded scalar value from parent object in handle_value

Signed-off-by: Angadi Yashaswini <angadi@digiscrypt.com>

---------

Signed-off-by: Angadi Yashaswini <angadi@digiscrypt.com>

include/nlohmann/detail/input/json_sax.hpp

@@ -626,14 +626,7 @@ class json_sax_dom_callback_parser
         if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured())
         {
             // remove discarded value
-            for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it)
+            remove_discarded_value(*ref_stack.back());
-            {
-                if (it->is_discarded())
-                {
-                    ref_stack.back()->erase(it);
-                    break;
-                }
-            }
         }
 
         return true;
@@ -674,8 +667,9 @@ class json_sax_dom_callback_parser
     bool end_array()
     {
         bool keep = true;
+        const bool stored = ref_stack.back() != nullptr;
 
-        if (ref_stack.back())
+        if (stored)
         {
             keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back());
             if (keep)
@@ -709,9 +703,19 @@ class json_sax_dom_callback_parser
         keep_stack.pop_back();
 
         // remove discarded value
-        if (!keep && !ref_stack.empty() && ref_stack.back()->is_array())
+        if (!ref_stack.empty() && ref_stack.back())
         {
-            ref_stack.back()->m_data.m_value.array->pop_back();
+            if (!keep && ref_stack.back()->is_array())
+            {
+                ref_stack.back()->m_data.m_value.array->pop_back();
+            }
+            else if ((!keep || !stored) && ref_stack.back()->is_object())
+            {
+                // the array is either still stored under its key or was never
+                // stored, leaving the placeholder key() wrote; both show up as
+                // a discarded member of the parent object
+                remove_discarded_value(*ref_stack.back());
+            }
         }
 
         return true;
@@ -801,6 +805,19 @@ class json_sax_dom_callback_parser
     }
 #endif
 
+    /// remove the discarded value the callback rejected from its parent
+    static void remove_discarded_value(BasicJsonType& parent)
+    {
+        for (auto it = parent.begin(); it != parent.end(); ++it)
+        {
+            if (it->is_discarded())
+            {
+                parent.erase(it);
+                break;
+            }
+        }
+    }
+
     /*!
     @param[in] v  value to add to the JSON value we build during parsing
     @param[in] skip_callback  whether we should skip calling the callback
@@ -841,6 +858,18 @@ class json_sax_dom_callback_parser
         // do not handle this value if we just learnt it shall be discarded
         if (!keep)
         {
+            // if the value was to become an object member, key() already
+            // stored a placeholder for it that has to be removed again
+            if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object())
+            {
+                JSON_ASSERT(!key_keep_stack.empty());
+                const bool placeholder_stored = key_keep_stack.back();
+                key_keep_stack.pop_back();
+                if (placeholder_stored)
+                {
+                    remove_discarded_value(*ref_stack.back());
+                }
+            }
             return {false, nullptr};
         }
 

single_include/nlohmann/json.hpp

@@ -10065,14 +10065,7 @@ class json_sax_dom_callback_parser
         if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured())
         {
             // remove discarded value
-            for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it)
+            remove_discarded_value(*ref_stack.back());
-            {
-                if (it->is_discarded())
-                {
-                    ref_stack.back()->erase(it);
-                    break;
-                }
-            }
         }
 
         return true;
@@ -10113,8 +10106,9 @@ class json_sax_dom_callback_parser
     bool end_array()
     {
         bool keep = true;
+        const bool stored = ref_stack.back() != nullptr;
 
-        if (ref_stack.back())
+        if (stored)
         {
             keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back());
             if (keep)
@@ -10148,9 +10142,19 @@ class json_sax_dom_callback_parser
         keep_stack.pop_back();
 
         // remove discarded value
-        if (!keep && !ref_stack.empty() && ref_stack.back()->is_array())
+        if (!ref_stack.empty() && ref_stack.back())
         {
-            ref_stack.back()->m_data.m_value.array->pop_back();
+            if (!keep && ref_stack.back()->is_array())
+            {
+                ref_stack.back()->m_data.m_value.array->pop_back();
+            }
+            else if ((!keep || !stored) && ref_stack.back()->is_object())
+            {
+                // the array is either still stored under its key or was never
+                // stored, leaving the placeholder key() wrote; both show up as
+                // a discarded member of the parent object
+                remove_discarded_value(*ref_stack.back());
+            }
         }
 
         return true;
@@ -10240,6 +10244,19 @@ class json_sax_dom_callback_parser
     }
 #endif
 
+    /// remove the discarded value the callback rejected from its parent
+    static void remove_discarded_value(BasicJsonType& parent)
+    {
+        for (auto it = parent.begin(); it != parent.end(); ++it)
+        {
+            if (it->is_discarded())
+            {
+                parent.erase(it);
+                break;
+            }
+        }
+    }
+
     /*!
     @param[in] v  value to add to the JSON value we build during parsing
     @param[in] skip_callback  whether we should skip calling the callback
@@ -10280,6 +10297,18 @@ class json_sax_dom_callback_parser
         // do not handle this value if we just learnt it shall be discarded
         if (!keep)
         {
+            // if the value was to become an object member, key() already
+            // stored a placeholder for it that has to be removed again
+            if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_object())
+            {
+                JSON_ASSERT(!key_keep_stack.empty());
+                const bool placeholder_stored = key_keep_stack.back();
+                key_keep_stack.pop_back();
+                if (placeholder_stored)
+                {
+                    remove_discarded_value(*ref_stack.back());
+                }
+            }
             return {false, nullptr};
         }
 

tests/src/unit-class_parser.cpp

@@ -1440,6 +1440,13 @@ TEST_CASE("parser class")
             ]
         )";
 
+        const auto* structured_object = R"(
+            {
+                "foo": [1, 2],
+                "bar": 3
+            }
+        )";
+
         SECTION("filter nothing")
         {
             const json j_object = json::parse(s_object, [](int /*unused*/, json::parse_event_t /*unused*/, const json& /*unused*/) noexcept
@@ -1515,6 +1522,48 @@ TEST_CASE("parser class")
             CHECK (j_filtered2 == json({1}));
         }
 
+        SECTION("filter array in object")
+        {
+            // the array is discarded once it is already stored under its key
+            const json j_filtered1 = json::parse(structured_object, [](int /*unused*/, json::parse_event_t e, const json& /*parsed*/) noexcept
+            {
+                return e != json::parse_event_t::array_end;
+            });
+
+            CHECK (j_filtered1 == json({{"bar", 3}}));
+
+            // the array is discarded before it is stored, leaving the
+            // placeholder the key event wrote
+            const json j_filtered2 = json::parse(structured_object, [](int /*unused*/, json::parse_event_t e, const json& /*parsed*/) noexcept
+            {
+                return e != json::parse_event_t::array_start;
+            });
+
+            CHECK (j_filtered2 == json({{"bar", 3}}));
+        }
+
+        SECTION("filter value in object")
+        {
+            // the value is discarded after its key was kept, leaving the
+            // placeholder the key event wrote
+            const json j_filtered1 = json::parse(structured_object, [](int /*unused*/, json::parse_event_t e, const json & parsed) noexcept
+            {
+                return !(e == json::parse_event_t::value && parsed == json(3));
+            });
+
+            CHECK (j_filtered1 == json({{"foo", {1, 2}}}));
+
+            // the same value is discarded together with its key, so no
+            // placeholder was stored for it
+            const json j_filtered2 = json::parse(structured_object, [](int /*unused*/, json::parse_event_t e, const json & parsed) noexcept
+            {
+                return !((e == json::parse_event_t::key && parsed == json("bar")) ||
+                         (e == json::parse_event_t::value && parsed == json(3)));
+            });
+
+            CHECK (j_filtered2 == json({{"foo", {1, 2}}}));
+        }
+
         SECTION("filter specific events")
         {
             SECTION("first closing event")