#!/usr/bin/env python3 # __ _____ _____ _____ # __| | __| | | | JSON for Modern C++ (supporting code) # | | |__ | | | | | | version 3.12.0 # |_____|_____|_____|_|___| https://github.com/nlohmann/json # # SPDX-FileCopyrightText: 2013-2026 Niels Lohmann # SPDX-License-Identifier: MIT """Time one fixed benchmark against the json.hpp of every commit that changed it. Usage: history.py build --repo REPO --work DIR [--from v3.12.0] [--to origin/develop] [--extra INCLUDE_DIR] history.py run --work DIR [--rounds 4] history.py summarize --work DIR [--repo REPO] > history.csv build: lists the first-parent commits between --from and --to that changed single_include/nlohmann/json.hpp or json_view.hpp, extracts single_include/ of each (and of --from itself), and compiles history.cpp against it into DIR/bin/NNN, NNN being the line of the commit in DIR/commits.txt. --extra adds one more header directory at the end, for example an amalgamation of develop with open pull requests merged in. It also writes the input files to DIR/data (the nativejson-benchmark files from the test data, which must be in DIR/data already, plus a minified copy of citm_catalog.json). run: runs every binary once per round, in a new random order each round, and only while the 1-minute load average is below 4. Output: DIR/res/NNN-R.txt. summarize: the median over the rounds per binary and workload, as CSV with the commit, its date and its pull request. Only the Python 3 standard library is used; a C++17 compiler (c++) and git are needed. """ import argparse import concurrent.futures import csv import json import os import random import re import statistics import subprocess import sys import time from pathlib import Path HERE = Path(__file__).resolve().parent FLAGS = ["-std=c++17", "-O3", "-DNDEBUG", "-w"] def git(repo, *args): return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout def build_one(work, idx, rev, repo): hdr = work / "hdr" / idx hdr.mkdir(parents=True, exist_ok=True) if rev.startswith("dir:"): subprocess.run(["cp", "-R", rev[4:] + "/nlohmann", str(hdr)], check=True) else: archive = subprocess.run(["git", "-C", str(repo), "archive", rev, "single_include/nlohmann"], check=True, capture_output=True).stdout subprocess.run(["tar", "-x", "-C", str(hdr), "--strip-components=1"], input=archive, check=True) out = work / "bin" / idx cmd = ["c++", *FLAGS, "-I", str(hdr), str(HERE / "history.cpp"), "-o", str(out)] # json_view.hpp of the first commits that had it may not compile with history.cpp: time json.hpp alone then if subprocess.run(cmd, capture_output=True).returncode != 0: if subprocess.run(cmd[:-4] + ["-DHISTORY_NO_VIEW"] + cmd[-4:], capture_output=True).returncode != 0: return "FAIL %s %s" % (idx, rev) return None def build(a): work, repo = Path(a.work), Path(a.repo) (work / "bin").mkdir(parents=True, exist_ok=True) revs = [a.__dict__["from"]] + git(repo, "log", "--first-parent", "--reverse", "--format=%H", "%s..%s" % (a.__dict__["from"], a.to), "--", "single_include/nlohmann/json.hpp", "single_include/nlohmann/json_view.hpp").split() if a.extra: revs.append("dir:" + str(Path(a.extra).resolve())) (work / "commits.txt").write_text("\n".join(revs) + "\n") data = work / "data" citm = json.loads((data / "citm_catalog.json").read_text(encoding="utf-8")) (data / "citm_compact.json").write_text(json.dumps(citm, separators=(",", ":"), ensure_ascii=False), encoding="utf-8") with concurrent.futures.ThreadPoolExecutor(os.cpu_count()) as ex: for r in ex.map(lambda p: build_one(work, "%03d" % p[0], p[1], repo), enumerate(revs)): if r: print(r, file=sys.stderr) def load(): out = subprocess.run(["sysctl", "-n", "vm.loadavg"], capture_output=True, text=True).stdout return float(out.split()[1]) if out else os.getloadavg()[0] def run(a): work = Path(a.work) (work / "res").mkdir(exist_ok=True) bins = sorted(p.name for p in (work / "bin").iterdir()) for r in range(1, a.rounds + 1): random.shuffle(bins) for b in bins: while load() >= 4: time.sleep(15) out = subprocess.run([str(work / "bin" / b), str(work / "data")], check=True, capture_output=True, text=True) (work / "res" / ("%s-%d.txt" % (b, r))).write_text(out.stdout) print("round %d done" % r, file=sys.stderr) def summarize(a): work = Path(a.work) revs = (work / "commits.txt").read_text().split() times, names = {}, [] for f in sorted((work / "res").glob("*.txt")): idx = int(f.name.split("-")[0]) for line in f.read_text().splitlines(): name, us = line.split() if name not in names: names.append(name) times.setdefault((idx, name), []).append(float(us)) w = csv.writer(sys.stdout, lineterminator="\n") w.writerow(["index", "commit", "date", "pr", "subject"] + names) for idx, rev in enumerate(revs): if rev.startswith("dir:"): sha, date, pr, subject = "candidate", "", "", "develop + open pull requests" else: sha, date, subject = git(a.repo, "log", "-1", "--format=%h%x09%cs%x09%s", rev).strip().split("\t") m = re.search(r"\(#(\d+)\)$", subject) pr = m.group(1) if m else "" w.writerow([idx, sha, date, pr, subject] + ["%.3f" % statistics.median(times[(idx, n)]) if (idx, n) in times else "" for n in names]) def main(): p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = p.add_subparsers(dest="cmd", required=True) b = sub.add_parser("build") b.add_argument("--repo", required=True) b.add_argument("--work", required=True) b.add_argument("--from", default="v3.12.0") b.add_argument("--to", default="origin/develop") b.add_argument("--extra") r = sub.add_parser("run") r.add_argument("--work", required=True) r.add_argument("--rounds", type=int, default=4) s = sub.add_parser("summarize") s.add_argument("--work", required=True) s.add_argument("--repo", default=str(HERE.parents[2])) a = p.parse_args() {"build": build, "run": run, "summarize": summarize}[a.cmd](a) if __name__ == "__main__": main()