json/tests/benchmarks/history/history.py

#!/usr/bin/env python3
#     __ _____ _____ _____
#  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
# |  |  |__   |  |  | | | |  version 3.12.0
# |_____|_____|_____|_|___|  https://github.com/nlohmann/json
#
# SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
# SPDX-License-Identifier: MIT

"""Time one fixed benchmark against the json.hpp of every commit that changed it.

Usage:
  history.py build     --repo REPO --work DIR [--from v3.12.0] [--to origin/develop] [--extra INCLUDE_DIR]
  history.py run       --work DIR [--rounds 4]
  history.py summarize --work DIR [--repo REPO] > history.csv

build: lists the first-parent commits between --from and --to that changed single_include/nlohmann/json.hpp or
json_view.hpp, extracts single_include/ of each (and of --from itself), and compiles history.cpp against it into
DIR/bin/NNN, NNN being the line of the commit in DIR/commits.txt. --extra adds one more header directory at the end,
for example an amalgamation of develop with open pull requests merged in. It also writes the input files to DIR/data
(the nativejson-benchmark files from the test data, which must be in DIR/data already, plus a minified copy of
citm_catalog.json).

run: runs every binary once per round, in a new random order each round, and only while the 1-minute load average is
below 4. Output: DIR/res/NNN-R.txt.

summarize: the median over the rounds per binary and workload, as CSV with the commit, its date and its pull request.

Only the Python 3 standard library is used; a C++17 compiler (c++) and git are needed.
"""
import argparse
import concurrent.futures
import csv
import json
import os
import random
import re
import statistics
import subprocess
import sys
import time
from pathlib import Path

HERE = Path(__file__).resolve().parent
FLAGS = ["-std=c++17", "-O3", "-DNDEBUG", "-w"]


def git(repo, *args):
    return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout


def build_one(work, idx, rev, repo):
    hdr = work / "hdr" / idx
    hdr.mkdir(parents=True, exist_ok=True)
    if rev.startswith("dir:"):
        subprocess.run(["cp", "-R", rev[4:] + "/nlohmann", str(hdr)], check=True)
    else:
        archive = subprocess.run(["git", "-C", str(repo), "archive", rev, "single_include/nlohmann"],
                                 check=True, capture_output=True).stdout
        subprocess.run(["tar", "-x", "-C", str(hdr), "--strip-components=1"], input=archive, check=True)
    out = work / "bin" / idx
    cmd = ["c++", *FLAGS, "-I", str(hdr), str(HERE / "history.cpp"), "-o", str(out)]
    # json_view.hpp of the first commits that had it may not compile with history.cpp: time json.hpp alone then
    if subprocess.run(cmd, capture_output=True).returncode != 0:
        if subprocess.run(cmd[:-4] + ["-DHISTORY_NO_VIEW"] + cmd[-4:], capture_output=True).returncode != 0:
            return "FAIL %s %s" % (idx, rev)
    return None


def build(a):
    work, repo = Path(a.work), Path(a.repo)
    (work / "bin").mkdir(parents=True, exist_ok=True)
    revs = [a.__dict__["from"]] + git(repo, "log", "--first-parent", "--reverse", "--format=%H",
                                      "%s..%s" % (a.__dict__["from"], a.to), "--",
                                      "single_include/nlohmann/json.hpp", "single_include/nlohmann/json_view.hpp").split()
    if a.extra:
        revs.append("dir:" + str(Path(a.extra).resolve()))
    (work / "commits.txt").write_text("\n".join(revs) + "\n")
    data = work / "data"
    citm = json.loads((data / "citm_catalog.json").read_text(encoding="utf-8"))
    (data / "citm_compact.json").write_text(json.dumps(citm, separators=(",", ":"), ensure_ascii=False), encoding="utf-8")
    with concurrent.futures.ThreadPoolExecutor(os.cpu_count()) as ex:
        for r in ex.map(lambda p: build_one(work, "%03d" % p[0], p[1], repo), enumerate(revs)):
            if r:
                print(r, file=sys.stderr)


def load():
    out = subprocess.run(["sysctl", "-n", "vm.loadavg"], capture_output=True, text=True).stdout
    return float(out.split()[1]) if out else os.getloadavg()[0]


def run(a):
    work = Path(a.work)
    (work / "res").mkdir(exist_ok=True)
    bins = sorted(p.name for p in (work / "bin").iterdir())
    for r in range(1, a.rounds + 1):
        random.shuffle(bins)
        for b in bins:
            while load() >= 4:
                time.sleep(15)
            out = subprocess.run([str(work / "bin" / b), str(work / "data")], check=True, capture_output=True, text=True)
            (work / "res" / ("%s-%d.txt" % (b, r))).write_text(out.stdout)
        print("round %d done" % r, file=sys.stderr)


def summarize(a):
    work = Path(a.work)
    revs = (work / "commits.txt").read_text().split()
    times, names = {}, []
    for f in sorted((work / "res").glob("*.txt")):
        idx = int(f.name.split("-")[0])
        for line in f.read_text().splitlines():
            name, us = line.split()
            if name not in names:
                names.append(name)
            times.setdefault((idx, name), []).append(float(us))
    w = csv.writer(sys.stdout, lineterminator="\n")
    w.writerow(["index", "commit", "date", "pr", "subject"] + names)
    for idx, rev in enumerate(revs):
        if rev.startswith("dir:"):
            sha, date, pr, subject = "candidate", "", "", "develop + open pull requests"
        else:
            sha, date, subject = git(a.repo, "log", "-1", "--format=%h%x09%cs%x09%s", rev).strip().split("\t")
            m = re.search(r"\(#(\d+)\)$", subject)
            pr = m.group(1) if m else ""
        w.writerow([idx, sha, date, pr, subject] +
                   ["%.3f" % statistics.median(times[(idx, n)]) if (idx, n) in times else "" for n in names])


def main():
    p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    sub = p.add_subparsers(dest="cmd", required=True)
    b = sub.add_parser("build")
    b.add_argument("--repo", required=True)
    b.add_argument("--work", required=True)
    b.add_argument("--from", default="v3.12.0")
    b.add_argument("--to", default="origin/develop")
    b.add_argument("--extra")
    r = sub.add_parser("run")
    r.add_argument("--work", required=True)
    r.add_argument("--rounds", type=int, default=4)
    s = sub.add_parser("summarize")
    s.add_argument("--work", required=True)
    s.add_argument("--repo", default=str(HERE.parents[2]))
    a = p.parse_args()
    {"build": build, "run": run, "summarize": summarize}[a.cmd](a)


if __name__ == "__main__":
    main()