Replace snprintf with a branch-free writer for \uXXXX escapes (#5235)

* Replace snprintf with a branch-free writer for \uXXXX escapes

dump_escaped called std::snprintf(..., "\u%04x", ...) once per escaped
code point in the string serialization hot path. snprintf re-parses
the format string and pulls in locale/printf machinery on every call,
which is far heavier than the fixed 6-/12-byte output warrants. This
is hot for any string containing control characters, and for all
non-ASCII text when ensure_ascii is set.

Replace it with write_u_escape, a small helper that writes the escape
directly into string_buffer via a nibble-to-hex lookup table, mirroring
the existing hand-rolled dump_integer fast path in the same file.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

* Fix clang-tidy avoid-c-arrays warning in write_u_escape

Use a const char* rather than a char[] lookup table, matching the
existing hex_bytes helper in the same file.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

* ♻️ adjust write_u_escape signature

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

---------

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://json.nlohmann.me/sitemap.xml

docs/mkdocs/hooks/copy_markdown_source.py

@@ -0,0 +1,25 @@
+"""Copy each documentation page's Markdown source into the built site."""
+
+# Creates a `<path>.md` sibling of each HTML output (for example,
+# `features/comments/` becomes `features/comments.md`) so agents and tools can
+# fetch the raw Markdown directly instead of parsing rendered HTML.
+
+import os
+import shutil
+
+_pages = []
+
+
+def on_files(files, config):
+    global _pages
+    _pages = [f for f in files if f.is_documentation_page()]
+    return files
+
+
+def on_post_build(config):
+    site_dir = config["site_dir"]
+    for file in _pages:
+        url = file.url.rstrip("/")
+        target = os.path.join(site_dir, (url or "index") + ".md")
+        os.makedirs(os.path.dirname(target), exist_ok=True)
+        shutil.copyfile(file.abs_src_path, target)

docs/mkdocs/mkdocs.yml

@@ -367,6 +367,9 @@ markdown_extensions:
       auto_append:
         - ../includes/glossary.md
 
+hooks:
+  - hooks/copy_markdown_source.py
+
 plugins:
   - search:
       separator: '[\s\-\.]'
@@ -389,6 +392,33 @@ plugins:
         - https://nlohmann.github.io/json/*
         - mailto:*
   - privacy
+  - llmstxt:
+      markdown_description: >
+        JSON for Modern C++ is a C++11 header-only library implementing a JSON
+        value type with an STL-like API, JSON Pointer/Patch, CBOR/MessagePack/
+        BSON/UBJSON/BJData binary format support, and a SAX-style parser interface.
+      sections:
+        Home:
+          - index.md
+          - home/*.md
+        Features:
+          - features/*.md
+          - features/binary_formats/*.md
+          - features/element_access/*.md
+          - features/parsing/*.md
+          - features/types/*.md
+        Integration:
+          - integration/*.md
+        API Documentation:
+          - api/*.md
+          - api/basic_json/*.md
+          - api/adl_serializer/*.md
+          - api/byte_container_with_subtype/*.md
+          - api/json_pointer/*.md
+          - api/json_sax/*.md
+          - api/macros/*.md
+        Community:
+          - community/*.md
 
 extra_css:
   - css/custom.css

docs/mkdocs/requirements.txt

@@ -7,5 +7,6 @@ mkdocs-material-extensions==1.3.1                   # extensions
 mkdocs-minify-plugin==0.8.0                         # plugin "minify"
 mkdocs-redirects==1.2.3                             # plugin "redirects"
 mkdocs-htmlproofer-plugin==1.5.0                    # plugin "htmlproofer"
+mkdocs-llmstxt==0.5.0                               # plugin "llmstxt"
 
 PyYAML==6.0.3                                       # linter

include/nlohmann/detail/output/serializer.hpp

@@ -465,18 +465,12 @@ class serializer
                             {
                                 if (codepoint <= 0xFFFF)
                                 {
-                                    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+                                    write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
-                                    static_cast<void>((std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x",
-                                                                      static_cast<std::uint16_t>(codepoint)));
-                                    bytes += 6;
                                 }
                                 else
                                 {
-                                    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+                                    write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
-                                    static_cast<void>((std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
+                                    write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
-                                                                      static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)),
-                                                                      static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu))));
-                                    bytes += 12;
                                 }
                             }
                             else
@@ -683,6 +677,32 @@ class serializer
         return result;
     }
 
+    /*!
+     * @brief write a lowercase "\uXXXX" escape sequence into @a string_buffer
+     *
+     * Branch-free replacement for `snprintf(buf, 7, "\\u%04x", codeunit)` in the
+     * string escaping hot path. It writes exactly six characters ('\\', 'u' and
+     * four hex digits) at position @a pos of @a string_buffer via a nibble
+     * lookup table, avoiding the format-string parsing and locale machinery of
+     * `snprintf`. Advances @a pos by the number of bytes written (6).
+     *
+     * @param[in] pos       position in @a string_buffer to write at; there must
+     *                      be at least 6 bytes of headroom
+     * @param[in] codeunit  16-bit value to encode
+     */
+    void write_u_escape(std::size_t& pos, std::uint16_t codeunit) noexcept
+    {
+        JSON_ASSERT(string_buffer.size() - pos >= 6);
+        constexpr const char* nibble_to_hex = "0123456789abcdef";
+        string_buffer[pos + 0] = '\\';
+        string_buffer[pos + 1] = 'u';
+        string_buffer[pos + 2] = nibble_to_hex[(codeunit >> 12u) & 0x0Fu];
+        string_buffer[pos + 3] = nibble_to_hex[(codeunit >> 8u) & 0x0Fu];
+        string_buffer[pos + 4] = nibble_to_hex[(codeunit >> 4u) & 0x0Fu];
+        string_buffer[pos + 5] = nibble_to_hex[codeunit & 0x0Fu];
+        pos += 6;
+    }
+
     // templates to avoid warnings about useless casts
     template <typename NumberType, enable_if_t<std::is_signed<NumberType>::value, int> = 0>
     bool is_negative_number(NumberType x)

single_include/nlohmann/json.hpp

@@ -19962,18 +19962,12 @@ class serializer
                             {
                                 if (codepoint <= 0xFFFF)
                                 {
-                                    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+                                    write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
-                                    static_cast<void>((std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x",
-                                                                      static_cast<std::uint16_t>(codepoint)));
-                                    bytes += 6;
                                 }
                                 else
                                 {
-                                    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+                                    write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
-                                    static_cast<void>((std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x",
+                                    write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
-                                                                      static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)),
-                                                                      static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu))));
-                                    bytes += 12;
                                 }
                             }
                             else
@@ -20180,6 +20174,32 @@ class serializer
         return result;
     }
 
+    /*!
+     * @brief write a lowercase "\uXXXX" escape sequence into @a string_buffer
+     *
+     * Branch-free replacement for `snprintf(buf, 7, "\\u%04x", codeunit)` in the
+     * string escaping hot path. It writes exactly six characters ('\\', 'u' and
+     * four hex digits) at position @a pos of @a string_buffer via a nibble
+     * lookup table, avoiding the format-string parsing and locale machinery of
+     * `snprintf`. Advances @a pos by the number of bytes written (6).
+     *
+     * @param[in] pos       position in @a string_buffer to write at; there must
+     *                      be at least 6 bytes of headroom
+     * @param[in] codeunit  16-bit value to encode
+     */
+    void write_u_escape(std::size_t& pos, std::uint16_t codeunit) noexcept
+    {
+        JSON_ASSERT(string_buffer.size() - pos >= 6);
+        constexpr const char* nibble_to_hex = "0123456789abcdef";
+        string_buffer[pos + 0] = '\\';
+        string_buffer[pos + 1] = 'u';
+        string_buffer[pos + 2] = nibble_to_hex[(codeunit >> 12u) & 0x0Fu];
+        string_buffer[pos + 3] = nibble_to_hex[(codeunit >> 8u) & 0x0Fu];
+        string_buffer[pos + 4] = nibble_to_hex[(codeunit >> 4u) & 0x0Fu];
+        string_buffer[pos + 5] = nibble_to_hex[codeunit & 0x0Fu];
+        pos += 6;
+    }
+
     // templates to avoid warnings about useless casts
     template <typename NumberType, enable_if_t<std::is_signed<NumberType>::value, int> = 0>
     bool is_negative_number(NumberType x)

tests/src/unit-convenience.cpp

@@ -168,6 +168,32 @@ TEST_CASE("convenience functions")
         CHECK_THROWS_WITH_AS(check_escaped("\xC2"), "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0xC2", json::type_error&);
     }
 
+    SECTION("string escape with ensure_ascii")
+    {
+        // control characters are escaped regardless of ensure_ascii
+        check_escaped("\x01", "\\u0001", true);
+        check_escaped("\x1f", "\\u001f", true);
+
+        // non-ASCII code points in the Basic Multilingual Plane are emitted as
+        // a single lowercase \uXXXX escape (exercises every nibble position)
+        check_escaped("\xC2\x80", "\\u0080", true);         // U+0080
+        check_escaped("\xC3\xBF", "\\u00ff", true);         // U+00FF (ÿ)
+        check_escaped("\xDF\xBF", "\\u07ff", true);         // U+07FF
+        check_escaped("\xE4\xBD\xA0", "\\u4f60", true);     // U+4F60 (你)
+        check_escaped("\xEA\xAF\x8D", "\\uabcd", true);     // U+ABCD
+        check_escaped("\xEF\xBF\xBD", "\\ufffd", true);     // U+FFFD (replacement char, all-f nibbles)
+
+        // code points outside the BMP are emitted as a UTF-16 surrogate pair
+        // of two lowercase \uXXXX escapes
+        check_escaped("\xF0\x90\x80\x80", "\\ud800\\udc00", true); // U+10000 (lowest astral)
+        check_escaped("\xF0\x9F\x98\x80", "\\ud83d\\ude00", true); // U+1F600 (😀)
+        check_escaped("\xF4\x8F\xBF\xBF", "\\udbff\\udfff", true); // U+10FFFF (highest code point)
+
+        // with ensure_ascii disabled, non-ASCII input is passed through verbatim
+        check_escaped("\xE4\xBD\xA0", "\xE4\xBD\xA0", false);
+        check_escaped("\xF0\x9F\x98\x80", "\xF0\x9F\x98\x80", false);
+    }
+
     SECTION("string concat")
     {
         using nlohmann::detail::concat;