Derive generate_natvis.py's ABI tag list and version from abi_macros.hpp

generate_natvis.py hard-coded abi_tags = ['_diag', '_ldvcmp', '_dp',
'_bics', '_psp', '_snul'] and required --version on the command line.
The source of truth is include/nlohmann/detail/abi_macros.hpp: the
NLOHMANN_JSON_ABI_TAG_* defines, the argument order of
NLOHMANN_JSON_ABI_TAGS_CONCAT, and NLOHMANN_JSON_VERSION_MAJOR/MINOR/
PATCH. Nothing checked that the copies stayed in sync, and they have
drifted apart before: _dp was added in #4517 but missed here until
#5544, and 3.11.3 shipped json_abi_v3_11_2 namespaces (#4340).

Parse the tag list (in NLOHMANN_JSON_ABI_TAGS_CONCAT order) and the
version from abi_macros.hpp instead of hard-coding them. Make
--version optional (falling back to the parsed version) and default
the output directory to the repository root the script lives in.

Add a "natvis" Makefile target that runs the script, and extend
check-amalgamation to regenerate nlohmann_json.natvis and fail on a
diff, the same way it already does for the amalgamated headers and
BUILD.bazel. Wire the same regeneration into check_amalgamation.yml,
using the tool copy checked out from develop (as the workflow already
does for amalgamate.py) and installing jinja2 from
tools/generate_natvis/requirements.txt. Update the tool's README to
say it must be re-run after adding an ABI tag or bumping the version.

Verified: a run against develop produces no diff (with either the
default or an explicit --version 3.12.0); adding a dummy
NLOHMANN_JSON_ABI_TAG_* without a matching #define makes the script
fail loudly instead of silently omitting the tag; xmllint --noout
passes on the regenerated file; and running the script from a
directory other than the one being checked (simulating the workflow's
separate tool checkout) against this repository root also produces no
diff.

Overlaps #5600, which added _ekmo to the same hand-written abi_tags
line and regenerated the file.

#5717 item 3

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/workflows/check_amalgamation.yml

@@ -35,6 +35,7 @@ jobs:
       MAIN_DIR: ${{ github.workspace }}/main
       INCLUDE_DIR: ${{ github.workspace }}/main/single_include/nlohmann
       TOOL_DIR: ${{ github.workspace }}/tools/tools/amalgamate
+      NATVIS_TOOL_DIR: ${{ github.workspace }}/tools/tools/generate_natvis
 
     steps:
       - name: Harden Runner
@@ -61,6 +62,9 @@ jobs:
           python3 -mvenv venv
           venv/bin/pip3 install -r $MAIN_DIR/tools/astyle/requirements.txt
 
+      - name: Install generate_natvis dependencies
+        run: pip3 install -r $NATVIS_TOOL_DIR/requirements.txt
+
       - name: Regenerate amalgamation, formatting, and BUILD.bazel
         run: |
           cd $MAIN_DIR
@@ -87,6 +91,9 @@ jobs:
           ${{ github.workspace }}/venv/bin/astyle --project=tools/astyle/.astylerc --suffix=none --quiet \
             $(find $SOURCE_DIRS -type f \( -name '*.hpp' -o -name '*.cpp' -o -name '*.cu' \) -not -path 'tests/thirdparty/*' -not -path 'tests/abi/include/nlohmann/*' | sort)
 
+      - name: Regenerate nlohmann_json.natvis
+        run: python3 $NATVIS_TOOL_DIR/generate_natvis.py $MAIN_DIR
+
       - name: Build patch and check for differences
         id: diff
         run: |

Makefile

@@ -1,4 +1,4 @@
-.PHONY: pretty clean ChangeLog.md release update_hedley update_hedley_undef BUILD.bazel
+.PHONY: pretty clean ChangeLog.md release update_hedley update_hedley_undef BUILD.bazel natvis
 
 ##########################################################################
 # configuration
@@ -35,6 +35,7 @@ all:
 	@echo "check-amalgamation - check whether sources have been amalgamated and BUILD.bazel is up to date"
 	@echo "clean - remove built files"
 	@echo "fuzzing - see tests/fuzzing.md for how to build and run the fuzzers"
+	@echo "natvis - regenerate nlohmann_json.natvis from the current ABI tags and version"
 	@echo "pretty - beautify code with Artistic Style"
 	@echo "run_benchmarks - build and run benchmarks"
 	@echo "update_hedley - download Hedley and regenerate hedley.hpp / hedley_undef.hpp"
@@ -94,6 +95,10 @@ $(AMALGAMATED_FILE): $(SRCS)
 $(AMALGAMATED_FWD_FILE): $(SRCS)
 	tools/amalgamate/amalgamate.py -c tools/amalgamate/config_json_fwd.json -s . --verbose=yes
 
+# regenerate nlohmann_json.natvis from the ABI tags and version in include/nlohmann/detail/abi_macros.hpp
+natvis:
+	python3 tools/generate_natvis/generate_natvis.py .
+
 # check if file single_include/nlohmann/json.hpp has been amalgamated from the nlohmann sources
 check-amalgamation:
 	@mv $(AMALGAMATED_FILE) $(AMALGAMATED_FILE)~
@@ -107,6 +112,10 @@ check-amalgamation:
 	@$(MAKE) BUILD.bazel
 	@diff BUILD.bazel BUILD.bazel~ || (echo "===================================================================\n  BUILD.bazel is out of date! Please run 'make BUILD.bazel'.\n===================================================================" ; mv BUILD.bazel~ BUILD.bazel ; false)
 	@mv BUILD.bazel~ BUILD.bazel
+	@mv nlohmann_json.natvis nlohmann_json.natvis~
+	@$(MAKE) natvis
+	@diff nlohmann_json.natvis nlohmann_json.natvis~ || (echo "===================================================================\n  nlohmann_json.natvis is out of date! Please run 'make natvis'.\n===================================================================" ; mv nlohmann_json.natvis~ nlohmann_json.natvis ; false)
+	@mv nlohmann_json.natvis~ nlohmann_json.natvis
 
 # generate the Bazel BUILD file; phony, because a removed header would not trigger a rebuild
 BUILD.bazel:

tools/generate_natvis/README.md

@@ -2,8 +2,26 @@
 
 Generate the Natvis debugger visualization file for all supported namespace combinations.
 
+The ABI tag list and the library version are parsed from
+`include/nlohmann/detail/abi_macros.hpp`, so this script must be re-run (via
+`make natvis`) whenever an `NLOHMANN_JSON_ABI_TAG_*` macro is added to that
+file or the library version is bumped — otherwise the committed
+`nlohmann_json.natvis` drifts from the header it visualizes, and
+`make check-amalgamation` fails.
+
 ## Usage
 
 ```shell
-./generate_natvis.py --version X.Y.Z output_directory/
+make natvis
 ```
+
+or, equivalently:
+
+```shell
+./generate_natvis.py [--version X.Y.Z] [repository_root/]
+```
+
+`--version` and the output/repository-root directory both default to values
+derived from this script's own location, so they only need to be given
+explicitly when generating a Natvis file for a different checkout or a
+version other than the one in `abi_macros.hpp`.

tools/generate_natvis/generate_natvis.py

@@ -7,21 +7,75 @@ import os
 import re
 import sys
 
+# Directory of the repository, assuming this script stays at
+# tools/generate_natvis/generate_natvis.py. Used only as the default value
+# for the "output" argument below.
+REPO_ROOT = os.path.normpath(os.path.join(sys.path[0], '..', '..'))
+
+
 def semver(v):
     if not re.fullmatch(r'\d+\.\d+\.\d+', v):
         raise ValueError
     return v
 
+
+def abi_info(repo_root):
+    """Derive the ABI tag list (in NLOHMANN_JSON_ABI_TAGS_CONCAT order) and the
+    library version from <repo_root>/include/nlohmann/detail/abi_macros.hpp,
+    so this script cannot drift from the header it visualizes."""
+    abi_macros_hpp = os.path.join(repo_root, 'include', 'nlohmann', 'detail', 'abi_macros.hpp')
+    with open(abi_macros_hpp) as f:
+        content = f.read()
+
+    # find the NLOHMANN_JSON_ABI_TAGS_CONCAT(...) invocation that lists the
+    # NLOHMANN_JSON_ABI_TAG_* identifiers in order (not its own #define, which
+    # only names its formal parameters a, b, c, ...)
+    tag_idents = None
+    for args in re.findall(r'NLOHMANN_JSON_ABI_TAGS_CONCAT\(\s*(.*?)\)', content, re.S):
+        idents = re.findall(r'NLOHMANN_JSON_ABI_TAG_\w+', args)
+        if idents:
+            tag_idents = idents
+            break
+    if not tag_idents:
+        raise ValueError(f'could not find NLOHMANN_JSON_ABI_TAGS_CONCAT(...) in {abi_macros_hpp}')
+
+    abi_tags = []
+    for ident in tag_idents:
+        # each tag is #define'd to its suffix (e.g. _diag) when the matching
+        # JSON_* option is enabled, and to nothing in the #else branch; only
+        # the non-empty definition matches here
+        match = re.search(r'#define\s+' + re.escape(ident) + r'\s+(_\w+)\s*\n', content)
+        if not match:
+            raise ValueError(f'could not find a non-empty #define for {ident} in {abi_macros_hpp}')
+        abi_tags.append(match.group(1))
+
+    version = {}
+    for part in ('MAJOR', 'MINOR', 'PATCH'):
+        match = re.search(r'#define\s+NLOHMANN_JSON_VERSION_' + part + r'\s+(\d+)', content)
+        if not match:
+            raise ValueError(f'could not find NLOHMANN_JSON_VERSION_{part} in {abi_macros_hpp}')
+        version[part] = match.group(1)
+
+    return abi_tags, '{MAJOR}.{MINOR}.{PATCH}'.format(**version)
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('--version', required=True, type=semver, help='Library version number')
-    parser.add_argument('output', help='Output directory for nlohmann_json.natvis')
+    parser.add_argument('--version', type=semver,
+                         help='Library version number (default: parsed from '
+                              'include/nlohmann/detail/abi_macros.hpp below "output")')
+    parser.add_argument('output', nargs='?', default=REPO_ROOT,
+                         help='Repository root: where include/nlohmann/detail/abi_macros.hpp is '
+                              'read from and where nlohmann_json.natvis is written '
+                              '(default: the repository root this script lives in)')
     args = parser.parse_args()
 
+    derived_tags, derived_version = abi_info(args.output)
+
     namespaces = ['nlohmann']
     abi_prefix = 'json_abi'
-    abi_tags = ['_diag', '_ldvcmp', '_dp', '_bics', '_psp', '_snul']
-    version = '_v' + args.version.replace('.', '_')
+    abi_tags = derived_tags
+    version = '_v' + (args.version or derived_version).replace('.', '_')
     inline_namespaces = []
 
     # generate all combinations of inline namespace names