diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index cd8088a5a..2f7a53964 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -828,14 +828,19 @@ indent_done: const auto idx = static_cast(emit(k, 0, 0, static_cast(p - b), 0) - base); if (depth != 0) { - const frame f = {cur_idx, cur_count, cur_is_object}; if (NLOHMANN_VIEW_LIKELY(depth <= 64)) { - cold.shallow[depth - 1] = f; + // field by field: a frame put together on the stack and + // copied would be read back wider than it was written, + // and that load waits until the stores are done + frame& f = cold.shallow[depth - 1]; + f.idx = cur_idx; + f.count = cur_count; + f.is_object = cur_is_object; } else { - cold.deep.push_back(f); + cold.deep.push_back(frame{cur_idx, cur_count, cur_is_object}); } } ++depth; @@ -851,19 +856,21 @@ indent_done: n.next = static_cast(out - base) - cur_idx; if (--depth != 0) { - frame f{}; if (NLOHMANN_VIEW_LIKELY(depth <= 64)) { - f = cold.shallow[depth - 1]; + const frame& f = cold.shallow[depth - 1]; + cur_idx = f.idx; + cur_count = f.count; + cur_is_object = f.is_object; } else { - f = cold.deep.back(); + const frame f = cold.deep.back(); cold.deep.pop_back(); + cur_idx = f.idx; + cur_count = f.count; + cur_is_object = f.is_object; } - cur_idx = f.idx; - cur_count = f.count; - cur_is_object = f.is_object; } } diff --git a/include/nlohmann/detail/view/scan.hpp b/include/nlohmann/detail/view/scan.hpp index 1a596ab81..427a53a10 100644 --- a/include/nlohmann/detail/view/scan.hpp +++ b/include/nlohmann/detail/view/scan.hpp @@ -67,13 +67,19 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint16_t load16(const unsigned char* p) noexcep /// in predicted branches: 16 for keys, whose lengths repeat from record to /// record, and 8 for string values (Value) where a vector loop follows, as /// their lengths vary more. Longer runs continue 16 bytes at a time with NEON -/// or SSE2, else eight bytes at a time. +/// or SSE2, else eight bytes at a time. With SSE2, the run is checked 16 bytes +/// at a time from its first byte instead: on x86-64, one compare that finds +/// the end of most keys and short values is faster than a branch per byte (on +/// AArch64, where a NEON mask costs more and branches predict well, slower). template NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* scan_string_run(const unsigned char* p, const unsigned char* e) noexcept { const std::uint8_t* plain = string_plain(); for (;;) { +#if NLOHMANN_VIEW_SSE2 + p = vector_plain_run(p, e); +#else if (e - p >= 16) { #define NLOHMANN_VIEW_STEP(i) if (NLOHMANN_VIEW_LIKELY(plain[p[i]] != 0)) {} else { p += (i); goto stop; } @@ -107,6 +113,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* scan_string_run(const unsigned #endif continue; } +#endif while (p != e && plain[*p] != 0) { ++p; @@ -115,7 +122,9 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* scan_string_run(const unsigned { return p; } +#if !NLOHMANN_VIEW_SSE2 stop: +#endif if (*p < 0x80) { return p; // quote, backslash, or control character diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index a85441d7e..1121a97fe 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -917,13 +917,19 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint16_t load16(const unsigned char* p) noexcep /// in predicted branches: 16 for keys, whose lengths repeat from record to /// record, and 8 for string values (Value) where a vector loop follows, as /// their lengths vary more. Longer runs continue 16 bytes at a time with NEON -/// or SSE2, else eight bytes at a time. +/// or SSE2, else eight bytes at a time. With SSE2, the run is checked 16 bytes +/// at a time from its first byte instead: on x86-64, one compare that finds +/// the end of most keys and short values is faster than a branch per byte (on +/// AArch64, where a NEON mask costs more and branches predict well, slower). template NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* scan_string_run(const unsigned char* p, const unsigned char* e) noexcept { const std::uint8_t* plain = string_plain(); for (;;) { +#if NLOHMANN_VIEW_SSE2 + p = vector_plain_run(p, e); +#else if (e - p >= 16) { #define NLOHMANN_VIEW_STEP(i) if (NLOHMANN_VIEW_LIKELY(plain[p[i]] != 0)) {} else { p += (i); goto stop; } @@ -957,6 +963,7 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* scan_string_run(const unsigned #endif continue; } +#endif while (p != e && plain[*p] != 0) { ++p; @@ -965,7 +972,9 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* scan_string_run(const unsigned { return p; } +#if !NLOHMANN_VIEW_SSE2 stop: +#endif if (*p < 0x80) { return p; // quote, backslash, or control character @@ -1873,14 +1882,19 @@ indent_done: const auto idx = static_cast(emit(k, 0, 0, static_cast(p - b), 0) - base); if (depth != 0) { - const frame f = {cur_idx, cur_count, cur_is_object}; if (NLOHMANN_VIEW_LIKELY(depth <= 64)) { - cold.shallow[depth - 1] = f; + // field by field: a frame put together on the stack and + // copied would be read back wider than it was written, + // and that load waits until the stores are done + frame& f = cold.shallow[depth - 1]; + f.idx = cur_idx; + f.count = cur_count; + f.is_object = cur_is_object; } else { - cold.deep.push_back(f); + cold.deep.push_back(frame{cur_idx, cur_count, cur_is_object}); } } ++depth; @@ -1896,19 +1910,21 @@ indent_done: n.next = static_cast(out - base) - cur_idx; if (--depth != 0) { - frame f{}; if (NLOHMANN_VIEW_LIKELY(depth <= 64)) { - f = cold.shallow[depth - 1]; + const frame& f = cold.shallow[depth - 1]; + cur_idx = f.idx; + cur_count = f.count; + cur_is_object = f.is_object; } else { - f = cold.deep.back(); + const frame f = cold.deep.back(); cold.deep.pop_back(); + cur_idx = f.idx; + cur_count = f.count; + cur_is_object = f.is_object; } - cur_idx = f.idx; - cur_count = f.count; - cur_is_object = f.is_object; } }