diff --git a/.github/workflows/check_amalgamation.yml b/.github/workflows/check_amalgamation.yml index 35f3a57f7..d81a80b18 100644 --- a/.github/workflows/check_amalgamation.yml +++ b/.github/workflows/check_amalgamation.yml @@ -35,6 +35,7 @@ jobs: MAIN_DIR: ${{ github.workspace }}/main INCLUDE_DIR: ${{ github.workspace }}/main/single_include/nlohmann TOOL_DIR: ${{ github.workspace }}/tools/tools/amalgamate + NATVIS_TOOL_DIR: ${{ github.workspace }}/tools/tools/generate_natvis steps: - name: Harden Runner @@ -61,6 +62,48 @@ jobs: python3 -mvenv venv venv/bin/pip3 install -r $MAIN_DIR/tools/astyle/requirements.txt + - name: Install generate_natvis dependencies + run: pip3 install -r $NATVIS_TOOL_DIR/requirements.txt + + - name: Regenerate the tools/macro_builder tables in macro_scope.hpp + run: | + cd $MAIN_DIR + + # Built from this PR's own tools/macro_builder/main.cpp, not a + # develop checkout: unlike amalgamate.py and generate_natvis.py, + # this tool has no other source of truth to check against (its + # own README documents that it must reproduce macro_scope.hpp + # byte for byte), so there is nothing to gain from checking out a + # separate copy, and doing so would make this step fail on a PR + # that adds support for a new dispatch table until that PR itself + # merges to develop, the same way generate_natvis.py's --version + # requirement briefly did. + TMPDIR=$(mktemp -d ./macro_builder_check.XXXXXX) + c++ -std=c++11 tools/macro_builder/main.cpp -o "$TMPDIR/macro_builder" + "$TMPDIR/macro_builder" > "$TMPDIR/paste.hpp" + "$TMPDIR/macro_builder" type_body > "$TMPDIR/type_body.hpp" + + # Splice the (still unindented) generated blocks back into + # macro_scope.hpp; the astyle pass below indents their + # continuation lines the same way it does for the rest of + # include/, so a correctly regenerated file comes out unchanged. + awk -v newfile="$TMPDIR/paste.hpp" ' + BEGIN { while ((getline line < newfile) > 0) { new = new line "\n" } } + /^#define NLOHMANN_JSON_EXPAND\( x \) x$/ { printf "%s", new; skip=1 } + skip && /^#define NLOHMANN_JSON_DOUBLE_PASTE63\(/ { skip=0; next } + skip { next } + { print } + ' include/nlohmann/detail/macro_scope.hpp > "$TMPDIR/macro_scope_1.hpp" + awk -v newfile="$TMPDIR/type_body.hpp" ' + BEGIN { while ((getline line < newfile) > 0) { new = new line "\n" } } + /^#define NLOHMANN_JSON_TYPE_BODY\(Prefix, \.\.\.\)/ { printf "%s", new; skip=1 } + skip && /^[[:space:]]*NLOHMANN_JSON_TYPE_BODY_SENTINEL\)\)$/ { skip=0; next } + skip { next } + { print } + ' "$TMPDIR/macro_scope_1.hpp" > "$TMPDIR/macro_scope_2.hpp" + mv "$TMPDIR/macro_scope_2.hpp" include/nlohmann/detail/macro_scope.hpp + rm -rf "$TMPDIR" + - name: Regenerate amalgamation, formatting, and BUILD.bazel run: | cd $MAIN_DIR @@ -88,6 +131,19 @@ jobs: ${{ github.workspace }}/venv/bin/astyle --project=tools/astyle/.astylerc --suffix=none --quiet \ $(find $SOURCE_DIRS -type f \( -name '*.hpp' -o -name '*.cpp' -o -name '*.cu' \) -not -path 'tests/thirdparty/*' -not -path 'tests/abi/include/nlohmann/*' | sort) + - name: Regenerate nlohmann_json.natvis + run: | + cd $MAIN_DIR + # Pass --version explicitly so this step also works with the tool + # copy from develop before this repository's own generate_natvis.py + # learns to derive the version itself: the older script requires + # --version, and the newer one accepts it as an explicit override. + ABI_MACROS=include/nlohmann/detail/abi_macros.hpp + VERSION_MAJOR=$(grep -m1 'define NLOHMANN_JSON_VERSION_MAJOR' $ABI_MACROS | grep -o '[0-9]\+') + VERSION_MINOR=$(grep -m1 'define NLOHMANN_JSON_VERSION_MINOR' $ABI_MACROS | grep -o '[0-9]\+') + VERSION_PATCH=$(grep -m1 'define NLOHMANN_JSON_VERSION_PATCH' $ABI_MACROS | grep -o '[0-9]\+') + python3 $NATVIS_TOOL_DIR/generate_natvis.py --version "$VERSION_MAJOR.$VERSION_MINOR.$VERSION_PATCH" $MAIN_DIR + - name: Build patch and check for differences id: diff run: | diff --git a/Makefile b/Makefile index ccea759af..fec936343 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: pretty clean ChangeLog.md release update_hedley update_hedley_undef BUILD.bazel +.PHONY: pretty clean ChangeLog.md release update_hedley update_hedley_undef BUILD.bazel natvis macro_builder_check ########################################################################## # configuration @@ -24,6 +24,9 @@ AMALGAMATED_FWD_FILE=single_include/nlohmann/json_fwd.hpp # json_literals.hpp only includes , so it is copied verbatim AMALGAMATED_LITERALS_FILE=single_include/nlohmann/json_literals.hpp +# the header with the argument-counting macros generated by tools/macro_builder +MACRO_SCOPE_HPP=include/nlohmann/detail/macro_scope.hpp + ########################################################################## # documentation of the Makefile's targets @@ -36,13 +39,9 @@ all: @echo "ChangeLog.md - generate ChangeLog file" @echo "check-amalgamation - check whether sources have been amalgamated and BUILD.bazel is up to date" @echo "clean - remove built files" - @echo "doctest - compile example files and check their output" - @echo "fuzz_testing - prepare fuzz testing of the JSON parser" - @echo "fuzz_testing_bon8 - prepare fuzz testing of the BON8 parser" - @echo "fuzz_testing_bson - prepare fuzz testing of the BSON parser" - @echo "fuzz_testing_cbor - prepare fuzz testing of the CBOR parser" - @echo "fuzz_testing_msgpack - prepare fuzz testing of the MessagePack parser" - @echo "fuzz_testing_ubjson - prepare fuzz testing of the UBJSON parser" + @echo "fuzzing - see tests/fuzzing.md for how to build and run the fuzzers" + @echo "macro_builder_check - check that macro_scope.hpp matches tools/macro_builder's output" + @echo "natvis - regenerate nlohmann_json.natvis from the current ABI tags and version" @echo "pretty - beautify code with Artistic Style" @echo "run_benchmarks - build and run benchmarks" @echo "update_hedley - download Hedley and regenerate hedley.hpp / hedley_undef.hpp" @@ -61,74 +60,6 @@ run_benchmarks: cd cmake-build-benchmarks ; ./json_benchmarks -########################################################################## -# fuzzing -########################################################################## - -# the overall fuzz testing target -fuzz_testing: - rm -fr fuzz-testing - mkdir -p fuzz-testing fuzz-testing/testcases fuzz-testing/out - $(MAKE) parse_afl_fuzzer -C tests CXX=afl-clang++ - mv tests/parse_afl_fuzzer fuzz-testing/fuzzer - find tests/data/json_tests -size -5k -name *json | xargs -I{} cp "{}" fuzz-testing/testcases - @echo "Execute: afl-fuzz -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer" - -fuzz_testing_bon8: - rm -fr fuzz-testing - mkdir -p fuzz-testing fuzz-testing/testcases fuzz-testing/out - $(MAKE) parse_bon8_fuzzer -C tests CXX=afl-clang++ - mv tests/parse_bon8_fuzzer fuzz-testing/fuzzer - find tests/data -size -5k -name *.bon8 | xargs -I{} cp "{}" fuzz-testing/testcases - @echo "Execute: afl-fuzz -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer" - -fuzz_testing_bson: - rm -fr fuzz-testing - mkdir -p fuzz-testing fuzz-testing/testcases fuzz-testing/out - $(MAKE) parse_bson_fuzzer -C tests CXX=afl-clang++ - mv tests/parse_bson_fuzzer fuzz-testing/fuzzer - find tests/data -size -5k -name *.bson | xargs -I{} cp "{}" fuzz-testing/testcases - @echo "Execute: afl-fuzz -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer" - -fuzz_testing_cbor: - rm -fr fuzz-testing - mkdir -p fuzz-testing fuzz-testing/testcases fuzz-testing/out - $(MAKE) parse_cbor_fuzzer -C tests CXX=afl-clang++ - mv tests/parse_cbor_fuzzer fuzz-testing/fuzzer - find tests/data -size -5k -name *.cbor | xargs -I{} cp "{}" fuzz-testing/testcases - @echo "Execute: afl-fuzz -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer" - -fuzz_testing_msgpack: - rm -fr fuzz-testing - mkdir -p fuzz-testing fuzz-testing/testcases fuzz-testing/out - $(MAKE) parse_msgpack_fuzzer -C tests CXX=afl-clang++ - mv tests/parse_msgpack_fuzzer fuzz-testing/fuzzer - find tests/data -size -5k -name *.msgpack | xargs -I{} cp "{}" fuzz-testing/testcases - @echo "Execute: afl-fuzz -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer" - -fuzz_testing_ubjson: - rm -fr fuzz-testing - mkdir -p fuzz-testing fuzz-testing/testcases fuzz-testing/out - $(MAKE) parse_ubjson_fuzzer -C tests CXX=afl-clang++ - mv tests/parse_ubjson_fuzzer fuzz-testing/fuzzer - find tests/data -size -5k -name *.ubjson | xargs -I{} cp "{}" fuzz-testing/testcases - @echo "Execute: afl-fuzz -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer" - -fuzzing-start: - afl-fuzz -S fuzzer1 -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer > /dev/null & - afl-fuzz -S fuzzer2 -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer > /dev/null & - afl-fuzz -S fuzzer3 -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer > /dev/null & - afl-fuzz -S fuzzer4 -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer > /dev/null & - afl-fuzz -S fuzzer5 -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer > /dev/null & - afl-fuzz -S fuzzer6 -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer > /dev/null & - afl-fuzz -S fuzzer7 -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer > /dev/null & - afl-fuzz -M fuzzer0 -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer - -fuzzing-stop: - -killall fuzzer - -killall afl-fuzz - - ########################################################################## # Static analysis ########################################################################## @@ -158,10 +89,6 @@ install_astyle: pretty: install_astyle $(ASTYLE) --project=tools/astyle/.astylerc $(SRCS) $(TESTS_SRCS) $(AMALGAMATED_FILE) $(AMALGAMATED_FWD_FILE) $(AMALGAMATED_LITERALS_FILE) docs/mkdocs/docs/examples/*.cpp -# call the Clang-Format on all source files -pretty_format: - for FILE in $(SRCS) $(TESTS_SRCS) $(AMALGAMATED_FILE) docs/mkdocs/docs/examples/*.cpp; do echo $$FILE; clang-format -i $$FILE; done - # create single header files and pretty print amalgamate: $(AMALGAMATED_FILE) $(AMALGAMATED_FWD_FILE) $(AMALGAMATED_LITERALS_FILE) $(MAKE) pretty @@ -178,8 +105,26 @@ $(AMALGAMATED_FWD_FILE): $(SRCS) $(AMALGAMATED_LITERALS_FILE): include/nlohmann/json_literals.hpp cp include/nlohmann/json_literals.hpp $(AMALGAMATED_LITERALS_FILE) +# regenerate nlohmann_json.natvis from the ABI tags and version in include/nlohmann/detail/abi_macros.hpp +natvis: + python3 tools/generate_natvis/generate_natvis.py . + +# regenerate the two tools/macro_builder blocks of $(MACRO_SCOPE_HPP) (see its README.md) and diff against the +# checked-in header; phony, because it never writes $(MACRO_SCOPE_HPP) itself +macro_builder_check: + @set -e; \ + TMPDIR=$$(mktemp -d ./macro_builder_check.XXXXXX); \ + trap 'rm -rf "$$TMPDIR"' EXIT; \ + $(CXX) -std=c++11 tools/macro_builder/main.cpp -o "$$TMPDIR/macro_builder"; \ + "$$TMPDIR/macro_builder" > "$$TMPDIR/paste.hpp"; \ + "$$TMPDIR/macro_builder" type_body > "$$TMPDIR/type_body.hpp"; \ + $(ASTYLE) --project=tools/astyle/.astylerc --suffix=none --quiet "$$TMPDIR/paste.hpp" "$$TMPDIR/type_body.hpp"; \ + sed -n '/^#define NLOHMANN_JSON_EXPAND( x ) x$$/,/^#define NLOHMANN_JSON_DOUBLE_PASTE63(/p' $(MACRO_SCOPE_HPP) > "$$TMPDIR/paste_actual.hpp"; \ + sed -n '/^#define NLOHMANN_JSON_TYPE_BODY(Prefix, \.\.\.)/,/^ NLOHMANN_JSON_TYPE_BODY_SENTINEL))$$/p' $(MACRO_SCOPE_HPP) > "$$TMPDIR/type_body_actual.hpp"; \ + diff "$$TMPDIR/paste.hpp" "$$TMPDIR/paste_actual.hpp" || (echo "===================================================================\n $(MACRO_SCOPE_HPP) (NLOHMANN_JSON_EXPAND..NLOHMANN_JSON_DOUBLE_PASTE63) is out of date!\n Regenerate it, see tools/macro_builder/README.md.\n===================================================================" ; exit 1); \ + diff "$$TMPDIR/type_body.hpp" "$$TMPDIR/type_body_actual.hpp" || (echo "===================================================================\n $(MACRO_SCOPE_HPP) (NLOHMANN_JSON_TYPE_BODY) is out of date!\n Regenerate it, see tools/macro_builder/README.md.\n===================================================================" ; exit 1) + # check if file single_include/nlohmann/json.hpp has been amalgamated from the nlohmann sources -# Note: this target is called by Travis check-amalgamation: @mv $(AMALGAMATED_FILE) $(AMALGAMATED_FILE)~ @mv $(AMALGAMATED_FWD_FILE) $(AMALGAMATED_FWD_FILE)~ @@ -195,6 +140,11 @@ check-amalgamation: @$(MAKE) BUILD.bazel @diff BUILD.bazel BUILD.bazel~ || (echo "===================================================================\n BUILD.bazel is out of date! Please run 'make BUILD.bazel'.\n===================================================================" ; mv BUILD.bazel~ BUILD.bazel ; false) @mv BUILD.bazel~ BUILD.bazel + @mv nlohmann_json.natvis nlohmann_json.natvis~ + @$(MAKE) natvis + @diff nlohmann_json.natvis nlohmann_json.natvis~ || (echo "===================================================================\n nlohmann_json.natvis is out of date! Please run 'make natvis'.\n===================================================================" ; mv nlohmann_json.natvis~ nlohmann_json.natvis ; false) + @mv nlohmann_json.natvis~ nlohmann_json.natvis + @$(MAKE) macro_builder_check # generate the Bazel BUILD file; phony, because a removed header would not trigger a rebuild BUILD.bazel: @@ -246,7 +196,7 @@ release: include.zip json.tar.xz cp $(AMALGAMATED_FWD_FILE) release_files cp $(AMALGAMATED_LITERALS_FILE) release_files mv $(AMALGAMATED_FILE).asc $(AMALGAMATED_FWD_FILE).asc $(AMALGAMATED_LITERALS_FILE).asc json.tar.xz json.tar.xz.asc include.zip include.zip.asc release_files - cd release_files ; shasum -a 256 json.hpp include.zip json.tar.xz > hashes.txt + cd release_files ; shasum -a 256 $$(find . -type f -not -name '*.asc' | sed 's|^\./||' | sort) > hashes.txt ########################################################################## @@ -256,7 +206,6 @@ release: include.zip json.tar.xz # clean up clean: rm -fr fuzz fuzz-testing *.dSYM tests/*.dSYM - rm -fr benchmarks/files/numbers/*.json rm -fr cmake-build-benchmarks fuzz-testing cmake-build-pvs-studio release_files $(MAKE) clean -Cdocs diff --git a/include/nlohmann/detail/macro_scope.hpp b/include/nlohmann/detail/macro_scope.hpp index 2d0fccf18..7c5ae3089 100644 --- a/include/nlohmann/detail/macro_scope.hpp +++ b/include/nlohmann/detail/macro_scope.hpp @@ -359,6 +359,7 @@ void templated_json_throw(ExceptionType exception) // Macros to simplify conversion from/to types +// NLOHMANN_JSON_EXPAND to NLOHMANN_JSON_DOUBLE_PASTE63 are generated by tools/macro_builder (see its README.md) #define NLOHMANN_JSON_EXPAND( x ) x #define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME #define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ @@ -621,6 +622,10 @@ void templated_json_throw(ExceptionType exception) // arguments, so dispatching on Type,BaseType,member... directly would run out // one slot early and cap the derived-type macros at 62 members instead of the // 63 that NLOHMANN_JSON_PASTE supports. +// +// The slot table below (down to the closing NLOHMANN_JSON_TYPE_BODY_SENTINEL)) +// is generated by tools/macro_builder (see its README.md; run with the +// "type_body" argument). #define NLOHMANN_JSON_TYPE_BODY(Prefix, ...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, \ Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, \ diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1fa63f344..3437b5408 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -2772,6 +2772,7 @@ void templated_json_throw(ExceptionType exception) // Macros to simplify conversion from/to types +// NLOHMANN_JSON_EXPAND to NLOHMANN_JSON_DOUBLE_PASTE63 are generated by tools/macro_builder (see its README.md) #define NLOHMANN_JSON_EXPAND( x ) x #define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME #define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ @@ -3034,6 +3035,10 @@ void templated_json_throw(ExceptionType exception) // arguments, so dispatching on Type,BaseType,member... directly would run out // one slot early and cap the derived-type macros at 62 members instead of the // 63 that NLOHMANN_JSON_PASTE supports. +// +// The slot table below (down to the closing NLOHMANN_JSON_TYPE_BODY_SENTINEL)) +// is generated by tools/macro_builder (see its README.md; run with the +// "type_body" argument). #define NLOHMANN_JSON_TYPE_BODY(Prefix, ...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, \ Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, Prefix ## MEMBERS, \ diff --git a/tools/amalgamate/README.md b/tools/amalgamate/README.md index 33ea9687b..658e57cbd 100644 --- a/tools/amalgamate/README.md +++ b/tools/amalgamate/README.md @@ -1,8 +1,8 @@ # amalgamate.py - Amalgamate C source and header files -Origin: https://bitbucket.org/erikedlund/amalgamate - -Mirror: https://github.com/edlund/amalgamate +Origin: https://github.com/edlund/amalgamate (formerly hosted at +https://bitbucket.org/erikedlund/amalgamate, which no longer exists; see +`CHANGES.md` for the upstream commit this copy is based on) `amalgamate.py` aims to make it easy to use SQLite-style C source and header amalgamation in projects. @@ -41,21 +41,22 @@ results. ## Installing amalgamate.py -Python v.2.7.0 or higher is required. +Python 3 is required. -`amalgamate.py` can be tested and installed using the following commands: - - ./test.sh && sudo -k cp ./amalgamate.py /usr/local/bin/ +In this repository, `amalgamate.py` is not installed separately; it is run in +place through `make amalgamate`, which calls it once for `json.hpp` and once +for `json_fwd.hpp` (see the root `Makefile`). ## Using amalgamate.py - amalgamate.py [-v] -c path/to/config.json -s path/to/source/dir \ - [-p path/to/prologue.(c|h)] + amalgamate.py -c path/to/config.json -s path/to/source/dir \ + [-p path/to/prologue.(c|h)] [--verbose=yes|no] * The `-c, --config` option should specify the path to a JSON config file which lists the source files, include paths and where to write the resulting - amalgamation. Have a look at `test/source.c.json` and `test/include.h.json` - to see two examples. + amalgamation. `config_json.json` and `config_json_fwd.json` in this + directory are the configs used for `json.hpp` and `json_fwd.hpp`; each + sets `target`, `sources` and `include_paths`. The optional `external` list names include paths that are kept as `#include` directives instead of being inlined, e.g. `["nlohmann/json.hpp"]` for a header @@ -68,3 +69,6 @@ Python v.2.7.0 or higher is required. * The `-p, --prologue` option should specify the path to a file which will be added to the beginning of the amalgamation. It is optional. + * The `-v, --verbose` option takes `yes` or `no` (for example + `--verbose=yes`, as used by the Makefile). It is optional. + diff --git a/tools/generate_natvis/README.md b/tools/generate_natvis/README.md index e11f29eec..3995758c2 100644 --- a/tools/generate_natvis/README.md +++ b/tools/generate_natvis/README.md @@ -2,8 +2,26 @@ Generate the Natvis debugger visualization file for all supported namespace combinations. +The ABI tag list and the library version are parsed from +`include/nlohmann/detail/abi_macros.hpp`, so this script must be re-run (via +`make natvis`) whenever an `NLOHMANN_JSON_ABI_TAG_*` macro is added to that +file or the library version is bumped — otherwise the committed +`nlohmann_json.natvis` drifts from the header it visualizes, and +`make check-amalgamation` fails. + ## Usage ```shell -./generate_natvis.py --version X.Y.Z output_directory/ +make natvis ``` + +or, equivalently: + +```shell +./generate_natvis.py [--version X.Y.Z] [repository_root/] +``` + +`--version` and the output/repository-root directory both default to values +derived from this script's own location, so they only need to be given +explicitly when generating a Natvis file for a different checkout or a +version other than the one in `abi_macros.hpp`. diff --git a/tools/generate_natvis/generate_natvis.py b/tools/generate_natvis/generate_natvis.py index 8562e8f1a..f4cd6fcd2 100755 --- a/tools/generate_natvis/generate_natvis.py +++ b/tools/generate_natvis/generate_natvis.py @@ -7,21 +7,75 @@ import os import re import sys +# Directory of the repository, assuming this script stays at +# tools/generate_natvis/generate_natvis.py. Used only as the default value +# for the "output" argument below. +REPO_ROOT = os.path.normpath(os.path.join(sys.path[0], '..', '..')) + + def semver(v): if not re.fullmatch(r'\d+\.\d+\.\d+', v): raise ValueError return v + +def abi_info(repo_root): + """Derive the ABI tag list (in NLOHMANN_JSON_ABI_TAGS_CONCAT order) and the + library version from /include/nlohmann/detail/abi_macros.hpp, + so this script cannot drift from the header it visualizes.""" + abi_macros_hpp = os.path.join(repo_root, 'include', 'nlohmann', 'detail', 'abi_macros.hpp') + with open(abi_macros_hpp) as f: + content = f.read() + + # find the NLOHMANN_JSON_ABI_TAGS_CONCAT(...) invocation that lists the + # NLOHMANN_JSON_ABI_TAG_* identifiers in order (not its own #define, which + # only names its formal parameters a, b, c, ...) + tag_idents = None + for args in re.findall(r'NLOHMANN_JSON_ABI_TAGS_CONCAT\(\s*(.*?)\)', content, re.S): + idents = re.findall(r'NLOHMANN_JSON_ABI_TAG_\w+', args) + if idents: + tag_idents = idents + break + if not tag_idents: + raise ValueError(f'could not find NLOHMANN_JSON_ABI_TAGS_CONCAT(...) in {abi_macros_hpp}') + + abi_tags = [] + for ident in tag_idents: + # each tag is #define'd to its suffix (e.g. _diag) when the matching + # JSON_* option is enabled, and to nothing in the #else branch; only + # the non-empty definition matches here + match = re.search(r'#define\s+' + re.escape(ident) + r'\s+(_\w+)\s*\n', content) + if not match: + raise ValueError(f'could not find a non-empty #define for {ident} in {abi_macros_hpp}') + abi_tags.append(match.group(1)) + + version = {} + for part in ('MAJOR', 'MINOR', 'PATCH'): + match = re.search(r'#define\s+NLOHMANN_JSON_VERSION_' + part + r'\s+(\d+)', content) + if not match: + raise ValueError(f'could not find NLOHMANN_JSON_VERSION_{part} in {abi_macros_hpp}') + version[part] = match.group(1) + + return abi_tags, '{MAJOR}.{MINOR}.{PATCH}'.format(**version) + + if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--version', required=True, type=semver, help='Library version number') - parser.add_argument('output', help='Output directory for nlohmann_json.natvis') + parser.add_argument('--version', type=semver, + help='Library version number (default: parsed from ' + 'include/nlohmann/detail/abi_macros.hpp below "output")') + parser.add_argument('output', nargs='?', default=REPO_ROOT, + help='Repository root: where include/nlohmann/detail/abi_macros.hpp is ' + 'read from and where nlohmann_json.natvis is written ' + '(default: the repository root this script lives in)') args = parser.parse_args() + derived_tags, derived_version = abi_info(args.output) + namespaces = ['nlohmann'] abi_prefix = 'json_abi' - abi_tags = ['_diag', '_ldvcmp', '_dp', '_bics', '_psp', '_snul'] - version = '_v' + args.version.replace('.', '_') + abi_tags = derived_tags + version = '_v' + (args.version or derived_version).replace('.', '_') inline_namespaces = [] # generate all combinations of inline namespace names diff --git a/tools/macro_builder/README.md b/tools/macro_builder/README.md new file mode 100644 index 000000000..f425f22c9 --- /dev/null +++ b/tools/macro_builder/README.md @@ -0,0 +1,51 @@ +# macro_builder + +Generates the argument-counting macros behind the `NLOHMANN_DEFINE_TYPE_*` and `NLOHMANN_DEFINE_DERIVED_TYPE_*` +macros in [`include/nlohmann/detail/macro_scope.hpp`](../../include/nlohmann/detail/macro_scope.hpp): + +- `NLOHMANN_JSON_EXPAND` +- `NLOHMANN_JSON_GET_MACRO`, which selects a macro by the number of its arguments (64 slots) +- `NLOHMANN_JSON_PASTE`, which calls a function-like macro for each member, and its helpers `NLOHMANN_JSON_PASTE2` + to `NLOHMANN_JSON_PASTE64` +- `NLOHMANN_JSON_DOUBLE_PASTE`, which the `*_WITH_NAMES` macros use to call a function-like macro for each + (JSON name, member) pair, and its helpers `NLOHMANN_JSON_DOUBLE_PASTE3` to `NLOHMANN_JSON_DOUBLE_PASTE63` +- the slot table of `NLOHMANN_JSON_TYPE_BODY`, which dispatches `NLOHMANN_DEFINE_TYPE_*(Type)` (no further + arguments) to the zero-member implementation and every other argument count to the one-or-more-member + implementation + +The number of slots (`max_args` in [`main.cpp`](main.cpp)) sets the member limit of these macros. +`NLOHMANN_JSON_PASTE` and `NLOHMANN_JSON_TYPE_BODY` take the function/prefix as their first argument, so 64 slots +allow 63 members; `NLOHMANN_JSON_DOUBLE_PASTE` additionally consumes its members two at a time (name, member), so +it only defines the odd helpers up to `NLOHMANN_JSON_DOUBLE_PASTE63`. + +## Usage + +From the project root: + +```shell +c++ -std=c++11 tools/macro_builder/main.cpp -o macro_builder +./macro_builder +./macro_builder type_body +``` + +1. Run `./macro_builder` (no arguments). In `include/nlohmann/detail/macro_scope.hpp`, replace the lines from + `#define NLOHMANN_JSON_EXPAND( x ) x` to the `#define NLOHMANN_JSON_DOUBLE_PASTE63(...)` line with the output, + without its trailing empty line. +2. Run `./macro_builder type_body`. Replace the lines from `#define NLOHMANN_JSON_TYPE_BODY(Prefix, ...)` to the + `NLOHMANN_JSON_TYPE_BODY_SENTINEL))` line with the output. +3. Run `make amalgamate`. It updates `single_include/nlohmann/json.hpp` and runs `make pretty`, which indents the + continuation lines that the tool writes unindented. + +With an unchanged `main.cpp`, these steps reproduce both blocks of `macro_scope.hpp` byte for byte. `make +macro_builder_check` (also run by CI, see `.github/workflows/check_amalgamation.yml`) automates this: it builds +`main.cpp`, regenerates both blocks, and fails on a diff against the checked-in header. + +## Maintained by hand + +The tool does not generate everything that depends on the number of slots. When changing `max_args`, also update: + +- the documented limit of 63 members in `docs/mkdocs/docs` and the tests at that limit in + `tests/src/unit-udt_macro.cpp` + +All three tables pass one macro name per slot to `NLOHMANN_JSON_GET_MACRO`, so they need exactly as many entries as +it has slots. diff --git a/tools/macro_builder/main.cpp b/tools/macro_builder/main.cpp index e676daacc..f035048f2 100644 --- a/tools/macro_builder/main.cpp +++ b/tools/macro_builder/main.cpp @@ -1,10 +1,14 @@ #include #include #include +#include using namespace std; -void build_code(int max_args) +// Builds NLOHMANN_JSON_EXPAND, NLOHMANN_JSON_GET_MACRO, and the +// NLOHMANN_JSON_PASTE / NLOHMANN_JSON_PASTE2..PASTE dispatch table +// and recursive definitions. +string build_paste_code(int max_args) { stringstream ss; ss << "#define NLOHMANN_JSON_EXPAND( x ) x" << endl; @@ -12,32 +16,93 @@ void build_code(int max_args) for (int i = 0 ; i < max_args ; i++) ss << "_" << i + 1 << ", "; ss << "NAME,...) NAME" << endl; - + ss << "#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \\" << endl; for (int i = max_args ; i > 1 ; i--) ss << "NLOHMANN_JSON_PASTE" << i << ", \\" << endl; ss << "NLOHMANN_JSON_PASTE1)(__VA_ARGS__))" << endl; - + ss << "#define NLOHMANN_JSON_PASTE2(func, v1) func(v1)" << endl; for (int i = 3 ; i <= max_args ; i++) { - ss << "#define NLOHMANN_JSON_PASTE" << i << "(func, "; + ss << "#define NLOHMANN_JSON_PASTE" << i << "(func, "; for (int j = 1 ; j < i -1 ; j++) - ss << "v" << j << ", "; + ss << "v" << j << ", "; ss << "v" << i-1 << ") NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE" << i-1 << "(func, "; for (int j = 2 ; j < i-1 ; j++) ss << "v" << j << ", "; ss << "v" << i-1 << ")" << endl; } - - cout << ss.str() << endl; + + return ss.str(); } -int main(int argc, char** argv) +// Builds the NLOHMANN_JSON_DOUBLE_PASTE dispatch table and recursive +// definitions used by the *_WITH_NAMES macros. Its GET_MACRO dispatch reuses +// the same max_args slots as NLOHMANN_JSON_PASTE, but DOUBLE_PASTE consumes +// its arguments two at a time (name, member), so an even slot count falls +// back to the next lower odd NLOHMANN_JSON_DOUBLE_PASTE. +string build_double_paste_code(int max_args) +{ + stringstream ss; + ss << "#define NLOHMANN_JSON_DOUBLE_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \\" << endl; + for (int i = max_args ; i > 1 ; i--) + { + int k = (i % 2 == 1) ? i : i - 1; + ss << "NLOHMANN_JSON_DOUBLE_PASTE" << k << ", \\" << endl; + } + ss << "NLOHMANN_JSON_DOUBLE_PASTE1)(__VA_ARGS__))" << endl; + + ss << "#define NLOHMANN_JSON_DOUBLE_PASTE3(func, v1, v2) func(v1, v2)" << endl; + for (int k = 5 ; k <= max_args - 1 ; k += 2) + { + ss << "#define NLOHMANN_JSON_DOUBLE_PASTE" << k << "(func, "; + for (int j = 1 ; j < k - 1 ; j++) + ss << "v" << j << ", "; + ss << "v" << k - 1 << ") NLOHMANN_JSON_DOUBLE_PASTE3(func, v1, v2) NLOHMANN_JSON_DOUBLE_PASTE" << k - 2 << "(func, "; + for (int j = 3 ; j < k - 1 ; j++) + ss << "v" << j << ", "; + ss << "v" << k - 1 << ")" << endl; + } + + return ss.str(); +} + +// Builds the NLOHMANN_JSON_TYPE_BODY dispatch table: max_args - 1 slots +// selecting the *_MEMBERS implementation and a final slot selecting +// *_EMPTY, so NLOHMANN_DEFINE_TYPE_*(Type) with no further arguments still +// resolves (issue #4041). +string build_type_body_table(int max_args) +{ + stringstream ss; + ss << "#define NLOHMANN_JSON_TYPE_BODY(Prefix, ...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \\" << endl; + const int per_line = 8; + for (int i = 1 ; i <= max_args ; i++) + { + ss << (i == max_args ? "Prefix ## EMPTY" : "Prefix ## MEMBERS") << ", "; + if (i % per_line == 0) + ss << "\\" << endl; + } + ss << "NLOHMANN_JSON_TYPE_BODY_SENTINEL))" << endl; + + return ss.str(); +} + +int main(int argc, char** argv) { int max_args = 64; - build_code(max_args); - + + // With "type_body", print only the NLOHMANN_JSON_TYPE_BODY dispatch + // table (a separate insertion point in macro_scope.hpp); otherwise + // print the EXPAND/GET_MACRO/PASTE/DOUBLE_PASTE block that precedes it. + if (argc > 1 && string(argv[1]) == "type_body") + { + cout << build_type_body_table(max_args); + } + else + { + cout << build_paste_code(max_args) << build_double_paste_code(max_args); + } + return 0; } - diff --git a/tools/serve_header/serve_header.py b/tools/serve_header/serve_header.py index 1f29cb589..5b5bf03f4 100755 --- a/tools/serve_header/serve_header.py +++ b/tools/serve_header/serve_header.py @@ -5,6 +5,8 @@ import logging import os import re import shutil +import socket +import ssl import sys import subprocess @@ -34,7 +36,7 @@ JSON_VERSION_RE = re.compile(r'\s*#\s*define\s+NLOHMANN_JSON_VERSION_MAJOR\s+') class ExitHandler(logging.StreamHandler): def __init__(self, level): - """.""" + """Exit the process on log records at or above level.""" super().__init__() self.level = level @@ -54,7 +56,7 @@ def is_project_root(test_dir='.'): class DirectoryEventBucket: def __init__(self, callback, delay=1.2, threshold=0.8): - """.""" + """Batch directory events and pass their common path to callback.""" self.delay = delay self.threshold = timedelta(seconds=threshold) self.callback = callback @@ -99,7 +101,7 @@ class WorkTree: make_command = 'make' def __init__(self, root_dir, tree_dir): - """.""" + """Track the working tree at tree_dir and its amalgamated header.""" self.root_dir = root_dir self.tree_dir = tree_dir self.rel_dir = os.path.relpath(tree_dir, root_dir) @@ -114,11 +116,11 @@ class WorkTree: self.build_time = t.strftime(DATETIME_FORMAT) def __hash__(self): - """.""" + """Hash by working tree directory.""" return hash((self.tree_dir)) def __eq__(self, other): - """.""" + """Compare by working tree directory.""" if not isinstance(other, type(self)): return NotImplemented return self.tree_dir == other.tree_dir @@ -150,7 +152,7 @@ class WorkTree: class WorkTrees(FileSystemEventHandler): def __init__(self, root_dir): - """.""" + """Find the working trees below root_dir and watch it for changes.""" super().__init__() self.root_dir = root_dir self.trees = set([]) @@ -250,11 +252,11 @@ class WorkTrees(FileSystemEventHandler): self.observer.stop() self.observer.join() -class HeaderRequestHandler(SimpleHTTPRequestHandler): # lgtm[py/missing-call-to-init] +class HeaderRequestHandler(SimpleHTTPRequestHandler): cors_origins = DEFAULT_CORS_ORIGINS def __init__(self, request, client_address, server): - """.""" + """Handle a request for a header below the working trees' root directory.""" self.worktrees = server.worktrees self.worktree = None try: @@ -336,7 +338,7 @@ class HeaderRequestHandler(SimpleHTTPRequestHandler): # lgtm[py/missing-call-to- class DualStackServer(ThreadingHTTPServer): def __init__(self, addr, worktrees): - """.""" + """Serve the headers of worktrees on addr.""" self.worktrees = worktrees super().__init__(addr, HeaderRequestHandler) @@ -349,8 +351,6 @@ class DualStackServer(ThreadingHTTPServer): if __name__ == '__main__': import argparse - import ssl - import socket import yaml # exit code