diff --git a/include/nlohmann/detail/view/object_index.hpp b/include/nlohmann/detail/view/object_index.hpp index 5c7551fe9..54a429f9c 100644 --- a/include/nlohmann/detail/view/object_index.hpp +++ b/include/nlohmann/detail/view/object_index.hpp @@ -47,24 +47,50 @@ constexpr std::size_t index_max_displacement = 64; static_assert(document_data::index_min_members >= 2 * index_max_displacement, "a table must be much larger than the probe window of a lookup"); -/// hash of a key: its bytes, eight at a time, in a fixed byte order +/// hash of a key: its bytes, eight at a time (in a fixed byte order), where the +/// last word may overlap the one before (so that no loop over single bytes is +/// needed); a key of fewer than eight bytes is read as two overlapping words of +/// four or two bytes (in the native byte order: the hash is not stored, and the +/// same on both sides of a lookup) inline std::uint64_t key_hash(const char* s, std::size_t n) noexcept { std::uint64_t h = 0x9E3779B97F4A7C15u * (n + 1); const auto* p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - while (n >= 8) - { - h = (h ^ read_eight_bytes(p)) * 0xBF58476D1CE4E5B9u; - h ^= h >> 29u; - p += 8; - n -= 8; - } std::uint64_t w = 0; - for (std::size_t i = 0; i < n; ++i) + if (n >= 8) { - w |= static_cast(p[i]) << (8u * i); + const unsigned char* const last = p + n - 8; + while (p < last) + { + h = (h ^ read_eight_bytes(p)) * 0xBF58476D1CE4E5B9u; + h ^= h >> 29u; + p += 8; + } + w = read_eight_bytes(last); } - h = (h ^ w) * 0x94D049BB133111EBu; + else if (n >= 4) + { + std::uint32_t head = 0; + std::uint32_t tail = 0; + std::memcpy(&head, p, 4); + std::memcpy(&tail, p + n - 4, 4); + w = static_cast(head) | (static_cast(tail) << 32u); + } + else if (n >= 2) + { + std::uint16_t head = 0; + std::uint16_t tail = 0; + std::memcpy(&head, p, 2); + std::memcpy(&tail, p + n - 2, 2); + w = static_cast(head) | (static_cast(tail) << 16u); + } + else if (n == 1) + { + w = p[0]; + } + // (a product only carries bits upwards, and the slot is taken from the low + // bits: the high bits of the last word are folded down before the multiplication) + h = (h ^ w ^ (w >> 29u)) * 0x94D049BB133111EBu; return h ^ (h >> 31u); } diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 7f7ec815f..ef5da68d7 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -3281,24 +3281,50 @@ constexpr std::size_t index_max_displacement = 64; static_assert(document_data::index_min_members >= 2 * index_max_displacement, "a table must be much larger than the probe window of a lookup"); -/// hash of a key: its bytes, eight at a time, in a fixed byte order +/// hash of a key: its bytes, eight at a time (in a fixed byte order), where the +/// last word may overlap the one before (so that no loop over single bytes is +/// needed); a key of fewer than eight bytes is read as two overlapping words of +/// four or two bytes (in the native byte order: the hash is not stored, and the +/// same on both sides of a lookup) inline std::uint64_t key_hash(const char* s, std::size_t n) noexcept { std::uint64_t h = 0x9E3779B97F4A7C15u * (n + 1); const auto* p = reinterpret_cast(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - while (n >= 8) - { - h = (h ^ read_eight_bytes(p)) * 0xBF58476D1CE4E5B9u; - h ^= h >> 29u; - p += 8; - n -= 8; - } std::uint64_t w = 0; - for (std::size_t i = 0; i < n; ++i) + if (n >= 8) { - w |= static_cast(p[i]) << (8u * i); + const unsigned char* const last = p + n - 8; + while (p < last) + { + h = (h ^ read_eight_bytes(p)) * 0xBF58476D1CE4E5B9u; + h ^= h >> 29u; + p += 8; + } + w = read_eight_bytes(last); } - h = (h ^ w) * 0x94D049BB133111EBu; + else if (n >= 4) + { + std::uint32_t head = 0; + std::uint32_t tail = 0; + std::memcpy(&head, p, 4); + std::memcpy(&tail, p + n - 4, 4); + w = static_cast(head) | (static_cast(tail) << 32u); + } + else if (n >= 2) + { + std::uint16_t head = 0; + std::uint16_t tail = 0; + std::memcpy(&head, p, 2); + std::memcpy(&tail, p + n - 2, 2); + w = static_cast(head) | (static_cast(tail) << 16u); + } + else if (n == 1) + { + w = p[0]; + } + // (a product only carries bits upwards, and the slot is taken from the low + // bits: the high bits of the last word are folded down before the multiplication) + h = (h ^ w ^ (w >> 29u)) * 0x94D049BB133111EBu; return h ^ (h >> 31u); } diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index 3b9da6b89..db2a5e2a1 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -21,14 +21,17 @@ using nlohmann::ordered_json_view; using json_view_test::generator; using json_view_test::has_duplicate_keys; +#include #include #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -598,6 +601,76 @@ TEST_CASE("json_view large objects") CHECK(without_collisions.memory_usage() >= with_collisions.memory_usage() + (table_size * sizeof(std::uint32_t))); } + SECTION("keys that differ in their last bytes are spread over the table") + { + // The hash must spread the usual families of keys (numbered names, hex + // strings, dates, short names): a key farther than index_max_displacement + // from its home slot would leave the object without a table. The keys + // are placed with linear probing, as the table does. + const auto max_displacement = [](const std::vector& keys) + { + std::size_t slots = 16; + while (slots < 2 * keys.size()) + { + slots *= 2; + } + std::vector used(slots, false); + std::size_t farthest = 0; + for (const std::string& key : keys) + { + std::size_t i = static_cast(nlohmann::detail::view::key_hash(key.data(), key.size())) & (slots - 1); + std::size_t distance = 0; + while (used[i]) + { + i = (i + 1) & (slots - 1); + ++distance; + } + used[i] = true; + farthest = (std::max)(farthest, distance); + } + return farthest; + }; + const auto numbered = [](const char* format, int count, int step) + { + std::vector keys; + for (int i = 0; i < count; ++i) + { + std::array buffer{}; + static_cast(std::snprintf(buffer.data(), buffer.size(), format, i * step)); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + keys.emplace_back(buffer.data()); + } + return keys; + }; + for (const char* format : + {"key%d", "%d", "item_%04d", "a%05d", "%03dxyz", "id%d", "user_%d", "%d_suffix", "0x%04x", "A%07d", "12345678%d", "1234567%d", "123456789%d", + "configuration_parameter_value_%04d_extra", "2026-01-%02dT00:00:00Z" + }) + { + CAPTURE(format) + CHECK(max_displacement(numbered(format, std::string(format).find("%02d") != std::string::npos ? 60 : 1000, 1)) < nlohmann::detail::view::index_max_displacement); + CHECK(max_displacement(numbered(format, 600, 7)) < nlohmann::detail::view::index_max_displacement); + } + std::mt19937_64 rng(7); + for (const std::size_t length : + { + std::size_t{1}, std::size_t{2}, std::size_t{3}, std::size_t{5}, std::size_t{7}, std::size_t{8}, std::size_t{9}, std::size_t{12}, std::size_t{16}, std::size_t{17}, std::size_t{33} + }) + { + CAPTURE(length) + std::set distinct; + while (distinct.size() < (length == 1 ? 26 : (length == 2 ? 500 : 700))) // (there are 26 and 676 keys of one and two letters) + { + std::string key(length, 'a'); + for (char& c : key) + { + c = static_cast('a' + (rng() % 26)); + } + distinct.insert(key); + } + CHECK(max_displacement(std::vector(distinct.begin(), distinct.end())) < nlohmann::detail::view::index_max_displacement); + } + } + SECTION("shrink_to_fit releases the tables' spare capacity") { const auto make_text = [](int objects, int members)