📡 make documentation more LLM friendly

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://json.nlohmann.me/sitemap.xml

docs/mkdocs/hooks/copy_markdown_source.py

@@ -0,0 +1,23 @@
+"""Copy each documentation page's Markdown source into the built site as a
+`<path>.md` sibling of its HTML output (e.g. features/comments/ -> features/comments.md),
+so agents/tools can fetch raw Markdown directly instead of parsing rendered HTML."""
+
+import os
+import shutil
+
+_pages = []
+
+
+def on_files(files, config):
+    global _pages
+    _pages = [f for f in files if f.is_documentation_page()]
+    return files
+
+
+def on_post_build(config):
+    site_dir = config["site_dir"]
+    for file in _pages:
+        url = file.url.rstrip("/")
+        target = os.path.join(site_dir, (url or "index") + ".md")
+        os.makedirs(os.path.dirname(target), exist_ok=True)
+        shutil.copyfile(file.abs_src_path, target)

docs/mkdocs/mkdocs.yml

@@ -367,6 +367,9 @@ markdown_extensions:
       auto_append:
         - ../includes/glossary.md
 
+hooks:
+  - hooks/copy_markdown_source.py
+
 plugins:
   - search:
       separator: '[\s\-\.]'
@@ -389,6 +392,33 @@ plugins:
         - https://nlohmann.github.io/json/*
         - mailto:*
   - privacy
+  - llmstxt:
+      markdown_description: >
+        JSON for Modern C++ is a C++11 header-only library implementing a JSON
+        value type with an STL-like API, JSON Pointer/Patch, CBOR/MessagePack/
+        BSON/UBJSON/BJData binary format support, and a SAX-style parser interface.
+      sections:
+        Home:
+          - index.md
+          - home/*.md
+        Features:
+          - features/*.md
+          - features/binary_formats/*.md
+          - features/element_access/*.md
+          - features/parsing/*.md
+          - features/types/*.md
+        Integration:
+          - integration/*.md
+        API Documentation:
+          - api/*.md
+          - api/basic_json/*.md
+          - api/adl_serializer/*.md
+          - api/byte_container_with_subtype/*.md
+          - api/json_pointer/*.md
+          - api/json_sax/*.md
+          - api/macros/*.md
+        Community:
+          - community/*.md
 
 extra_css:
   - css/custom.css

docs/mkdocs/requirements.txt

@@ -7,5 +7,6 @@ mkdocs-material-extensions==1.3.1                   # extensions
 mkdocs-minify-plugin==0.8.0                         # plugin "minify"
 mkdocs-redirects==1.2.3                             # plugin "redirects"
 mkdocs-htmlproofer-plugin==1.5.0                    # plugin "htmlproofer"
+mkdocs-llmstxt==0.5.0                               # plugin "llmstxt"
 
 PyYAML==6.0.3                                       # linter