mirror of
https://github.com/nlohmann/json.git
synced 2026-11-04 11:49:22 +00:00
docs: restructure the 3.13.0 post around themes and use cases
- deeper technical detail and code in the eight themes; pull request numbers move to the appendix, timeline steps are labelled by theme - comparison with other libraries by use case, as heatmaps per platform, with validation levels and the caveats next to the results - appendix: test files, all 182 suite rows with a histogram, timeline steps, algorithmic fixes, per-file tables, behaviour changes, setup, pull requests by theme Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
28 files changed
+33603
-908
No files matched your search
@@ -0,0 +1,183 @@
|
||||
benchmark,unit,3.12.0,3.13.0,speedup
|
||||
FromBinaryBuffer/bson / twitter,ns,3.07203e+06,998445,3.077
|
||||
ParseStringOrdered/jeopardy,ns,3.56038e+08,1.10829e+08,3.213
|
||||
JsonPointerToString/twitter,ns,2.16889e+06,793513,2.733
|
||||
BinaryToCbor/8192,ns,511.25,238.725,2.142
|
||||
FromMsgpack/unsigned_ints,ns,5.51164e+07,2.72444e+07,2.023
|
||||
JsonPointerAt/twitter,ns,780306,585055,1.334
|
||||
CopyOnly/twitter,ns,393964,399209,0.987
|
||||
FlattenOrdered/citm_catalog,ns,2.52682e+08,1.37388e+06,183.919
|
||||
FromBinaryFile/ubjson / canada,ns,1.34e+07,6.20075e+06,2.161
|
||||
FromBinaryShape/containers / bjdata,ns,1.41456e+07,7.04794e+06,2.007
|
||||
ParseIndented/jeopardy / 4,ns,3.33616e+08,1.31177e+08,2.543
|
||||
FromMsgpack/signed_ints,ns,6.19882e+07,3.19294e+07,1.941
|
||||
FromBinaryShape/containers / bson,ns,1.69936e+07,9.81169e+06,1.732
|
||||
BinaryToCbor/128,ns,247.24,110.472,2.238
|
||||
FromBinaryBuffer/cbor / jeopardy,ns,3.52319e+08,1.15873e+08,3.041
|
||||
ParseFile/jeopardy,ns,3.26899e+08,1.57231e+08,2.079
|
||||
FromMsgpack/jeopardy,ns,1.11847e+09,1.81305e+08,6.169
|
||||
FromBinaryFile/cbor / twitter,ns,9.70523e+06,1.5381e+06,6.310
|
||||
ToCbor/floats,ns,8.8461e+06,1.88426e+06,4.695
|
||||
ParseFile/small_signed_ints,ns,6.30771e+07,3.13826e+07,2.010
|
||||
FromBinaryBuffer/msgpack / twitter,ns,3.01937e+06,937570,3.220
|
||||
BinaryToCbor/16384,ns,947.242,408.275,2.320
|
||||
FromBinaryFile/bson / twitter,ns,9.98109e+06,1.78121e+06,5.604
|
||||
ParseString/twitter,ns,3.15572e+06,1.04655e+06,3.015
|
||||
Unflatten/twitter,ns,4.47709e+06,2.11403e+06,2.118
|
||||
BinaryToCbor/8,ns,242.891,46.6253,5.209
|
||||
BinaryToCbor/16,ns,238.431,46.5179,5.126
|
||||
Dump/small_signed_ints / 4,ns,2.49933e+07,8.62966e+06,2.896
|
||||
ParseString/jeopardy,ns,3.04927e+08,1.29515e+08,2.354
|
||||
Dump/signed_ints / 4,ns,3.18294e+07,1.68297e+07,1.891
|
||||
Flatten/twitter,ns,3.4886e+06,2.00532e+06,1.740
|
||||
FromBinaryBuffer/cbor / twitter,ns,3.00428e+06,964870,3.114
|
||||
Dump/jeopardy / 4,ns,2.19025e+08,3.43818e+07,6.370
|
||||
SaxParse/citm_catalog,ns,6.33865e+06,969665,6.537
|
||||
JsonPointerAt/citm_catalog,ns,1.80656e+06,1.10975e+06,1.628
|
||||
DestroyOnly/citm_catalog,ns,874073,663615,1.317
|
||||
FromBinaryShape/scalars / ubjson,ns,2.46643e+07,1.13906e+07,2.165
|
||||
Unflatten/canada,ns,4.04569e+07,1.07458e+07,3.765
|
||||
ParseString/floats,ns,1.18934e+08,3.02332e+07,3.934
|
||||
ParseIndented/canada / 4,ns,3.38495e+07,5.02409e+06,6.737
|
||||
ParseString/signed_ints,ns,1.19593e+08,3.14351e+07,3.804
|
||||
ParseFile/signed_ints,ns,1.24813e+08,4.61765e+07,2.703
|
||||
JsonPointerContains/twitter,ns,1.27938e+06,606069,2.111
|
||||
Dump/floats / 4,ns,7.10935e+07,2.41369e+07,2.945
|
||||
Dump/floats / -,ns,6.27641e+07,2.33213e+07,2.691
|
||||
ToCbor/canada,ns,1.36795e+06,546001,2.505
|
||||
FromBinaryBuffer/bson / canada,ns,9.21307e+06,5.51059e+06,1.672
|
||||
ToCbor/twitter,ns,638321,141975,4.496
|
||||
ParseIndented/twitter / 4,ns,3.58298e+06,1.10646e+06,3.238
|
||||
Iterate/citm_catalog,ns,297646,167302,1.779
|
||||
FromBinaryBuffer/bjdata / canada,ns,8.08015e+06,4.17334e+06,1.936
|
||||
ParseStreamValues/ndjson,ns,6.96248e+06,3.7254e+06,1.869
|
||||
Diff/twitter,ns,4.79296e+06,635531,7.542
|
||||
Dump/citm_catalog / -,ns,1.37753e+06,290658,4.739
|
||||
FromBinaryShape/nested / bjdata,ns,96017.5,37882.1,2.535
|
||||
ToJsonStrings/copy,ns,5.83243e+06,5.54046e+06,1.053
|
||||
Dump/canada / 4,ns,8.9039e+06,2.9943e+06,2.974
|
||||
CopyRing/twitter,ns,855374,766521,1.116
|
||||
FromBinaryBuffer/cbor / citm_catalog,ns,3.70397e+06,1.46121e+06,2.535
|
||||
FromBinaryFile/bjdata / twitter,ns,1.02688e+07,1.90492e+06,5.391
|
||||
FromBinaryShape/containers / ubjson_optimized,ns,1.4727e+07,4.80309e+06,3.066
|
||||
FromMsgpack/citm_catalog,ns,8.85701e+06,2.2473e+06,3.941
|
||||
CopyOnly/canada,ns,1.53425e+06,1.49609e+06,1.026
|
||||
Equal/canada,ns,453965,332795,1.364
|
||||
Diff/citm_catalog,ns,6.74369e+06,1.07135e+06,6.295
|
||||
ParseStringOrdered/canada,ns,1.49834e+07,4.29102e+06,3.492
|
||||
Flatten/canada,ns,2.82148e+07,1.11369e+07,2.533
|
||||
Accept/canada,ns,1.26836e+07,2.25933e+06,5.614
|
||||
CopyAssign/twitter,ns,804857,735341,1.095
|
||||
BinaryToCbor/4096,ns,402.442,159.22,2.528
|
||||
ParseFILE/citm_catalog,ns,3.45528e+07,7.26455e+06,4.756
|
||||
ToCbor/signed_ints,ns,1.13256e+07,2.76748e+06,4.092
|
||||
Patch/canada,ns,9.80719e+06,3.36006e+06,2.919
|
||||
GetArray<std::vector<int>>,ns,6589.63,655.021,10.060
|
||||
Dump/canada / -,ns,6.62927e+06,2.24661e+06,2.951
|
||||
GetVectorDouble/floats,ns,6.62214e+06,641259,10.327
|
||||
Accept/citm_catalog,ns,6.3596e+06,909397,6.993
|
||||
ParseFILE/twitter,ns,1.35627e+07,3.27982e+06,4.135
|
||||
FromBinaryBuffer/msgpack / jeopardy,ns,3.54302e+08,1.10166e+08,3.216
|
||||
ParseFile/floats,ns,1.3128e+08,4.88806e+07,2.686
|
||||
Accept/small_signed_ints,ns,5.5849e+07,1.88756e+07,2.959
|
||||
Accept/jeopardy,ns,2.2049e+08,6.3504e+07,3.472
|
||||
Accept/signed_ints,ns,1.15055e+08,2.84818e+07,4.040
|
||||
CopyRing/canada,ns,3.68758e+06,2.88527e+06,1.278
|
||||
Patch/twitter,ns,1.44963e+06,848729,1.708
|
||||
Iterate/twitter,ns,87147.2,61831.9,1.409
|
||||
ParseFile/twitter,ns,3.33795e+06,1.3545e+06,2.464
|
||||
GetArray<std::deque<int>>,ns,8110.78,1156.06,7.016
|
||||
FromBinaryBuffer/cbor / canada,ns,8.20468e+06,2.31071e+06,3.551
|
||||
Equal/twitter,ns,141057,125995,1.120
|
||||
ParseStreamValues/numbers,ns,1.87073e+06,370959,5.043
|
||||
Dump/small_signed_ints / -,ns,1.78162e+07,7.86562e+06,2.265
|
||||
JsonPointerToString/citm_catalog,ns,6.63798e+06,1.869e+06,3.552
|
||||
Copy/twitter,ns,797034,721889,1.104
|
||||
Diff/canada,ns,3.83337e+07,3.67301e+06,10.437
|
||||
FromBinaryBuffer/ubjson_optimized / twitter,ns,3.1281e+06,959992,3.258
|
||||
BinaryToCbor/32,ns,251.857,47.32,5.322
|
||||
GetArray<std::set<int>>,ns,58113.1,40911.2,1.420
|
||||
CopyAssign/citm_catalog,ns,1.66804e+06,1.48043e+06,1.127
|
||||
ToCbor/jeopardy,ns,7.16464e+07,2.11077e+07,3.394
|
||||
CopyRing/citm_catalog,ns,1.70236e+06,1.50767e+06,1.129
|
||||
FromBinaryBuffer/ubjson / canada,ns,8.14082e+06,4.15725e+06,1.958
|
||||
ToCbor/small_signed_ints,ns,1.02087e+07,2.3872e+06,4.276
|
||||
DestroyOnly/canada,ns,2.13545e+06,1.37061e+06,1.558
|
||||
FromBinaryBuffer/ubjson / citm_catalog,ns,3.81161e+06,1.73601e+06,2.196
|
||||
Dump/jeopardy / -,ns,2.01242e+08,3.04052e+07,6.619
|
||||
FromBinaryBuffer/msgpack / citm_catalog,ns,3.70428e+06,1.42385e+06,2.602
|
||||
ParseStringOrdered/citm_catalog,ns,1.07899e+07,1.63999e+06,6.579
|
||||
ParseString/unsigned_ints,ns,1.17897e+08,3.02538e+07,3.897
|
||||
Patch/citm_catalog,ns,2.48869e+06,1.60776e+06,1.548
|
||||
SaxParse/canada,ns,1.28389e+07,3.21707e+06,3.991
|
||||
ParseString/citm_catalog,ns,7.20303e+06,1.7454e+06,4.127
|
||||
FromBinaryFile/cbor / canada,ns,1.42056e+07,4.37844e+06,3.244
|
||||
BinaryToCbor/256,ns,269.641,86.3027,3.124
|
||||
ParseFILE/canada,ns,5.22098e+07,1.38907e+07,3.759
|
||||
BinaryToCbor/32768,ns,1538.42,806.578,1.907
|
||||
Accept/floats,ns,1.13316e+08,2.40029e+07,4.721
|
||||
FromMsgpack/floats,ns,5.92985e+07,2.71551e+07,2.184
|
||||
ParseString/small_signed_ints,ns,6.05098e+07,2.18687e+07,2.767
|
||||
JsonPointerParse/citm_catalog,ns,5.64871e+06,2.16902e+06,2.604
|
||||
Copy/canada,ns,3.64004e+06,2.8296e+06,1.286
|
||||
Dump/twitter / 4,ns,1.56293e+06,247785,6.308
|
||||
FromBinaryShape/nested / msgpack,ns,90351.2,34224,2.640
|
||||
FromBinaryBuffer/ubjson_optimized / canada,ns,8.36221e+06,2.82594e+06,2.959
|
||||
JsonPointerContains/citm_catalog,ns,2.71985e+06,1.23844e+06,2.196
|
||||
FromMsgpack/small_signed_ints,ns,6.27969e+07,3.33815e+07,1.881
|
||||
DestroyOnly/twitter,ns,401772,322543,1.246
|
||||
FromBinaryShape/containers / msgpack,ns,1.37471e+07,3.71417e+06,3.701
|
||||
FromBinaryShape/scalars / msgpack,ns,2.40944e+07,9.77821e+06,2.464
|
||||
ToCbor/citm_catalog,ns,1.02089e+06,257529,3.964
|
||||
GetVectorInt/signed_ints,ns,8.04308e+06,3.35704e+06,2.396
|
||||
Accept/twitter,ns,2.45943e+06,458446,5.365
|
||||
Iterate/canada,ns,861455,320474,2.688
|
||||
ToCbor/unsigned_ints,ns,7.98977e+06,2.75684e+06,2.898
|
||||
FromBinaryShape/scalars / bjdata,ns,2.43317e+07,1.12007e+07,2.172
|
||||
BinaryToCbor/2048,ns,332.826,114.084,2.917
|
||||
BinaryToCbor/512,ns,291.316,92.8039,3.139
|
||||
FlattenOrdered/twitter,ns,4.92534e+07,851500,57.843
|
||||
Copy/citm_catalog,ns,1.61348e+06,1.44942e+06,1.113
|
||||
FromBinaryShape/containers / ubjson,ns,1.40981e+07,6.87399e+06,2.051
|
||||
BinaryToCbor/1024,ns,304.119,95.8643,3.172
|
||||
FromBinaryBuffer/ubjson / twitter,ns,3.12147e+06,962822,3.242
|
||||
Dump/signed_ints / -,ns,2.34455e+07,1.61521e+07,1.452
|
||||
FromBinaryShape/scalars / cbor,ns,2.54779e+07,9.63099e+06,2.645
|
||||
FromBinaryBuffer/cbor / signed_ints,ns,3.57449e+07,1.2803e+07,2.792
|
||||
CopyOnly/citm_catalog,ns,734852,783826,0.938
|
||||
Dump/unsigned_ints / -,ns,2.3437e+07,1.6565e+07,1.415
|
||||
Accept/unsigned_ints,ns,1.14173e+08,2.74092e+07,4.165
|
||||
FromMsgpack/canada,ns,1.42472e+07,4.37022e+06,3.260
|
||||
ParseStringOrdered/twitter,ns,4.32143e+06,910717,4.745
|
||||
Dump/twitter / -,ns,1.38581e+06,207620,6.675
|
||||
BinaryToCbor/64,ns,250.506,81.5793,3.071
|
||||
JsonPointerParse/twitter,ns,2.40825e+06,979256,2.459
|
||||
GetMap/citm_catalog,ns,3.41415e+06,2.37109e+06,1.440
|
||||
ToJsonStrings/move,ns,5.83899e+06,3.02241e+06,1.932
|
||||
FromBinaryBuffer/ubjson / jeopardy,ns,3.67059e+08,1.15063e+08,3.190
|
||||
FromBinaryShape/nested / ubjson,ns,96068.6,37753.5,2.545
|
||||
FromBinaryBuffer/bjdata / twitter,ns,3.11851e+06,963022,3.238
|
||||
FromBinaryShape/nested / cbor,ns,93124.8,34434.7,2.704
|
||||
ParseFile/citm_catalog,ns,7.62707e+06,2.39694e+06,3.182
|
||||
FromBinaryFile/ubjson / twitter,ns,1.04355e+07,1.90842e+06,5.468
|
||||
Dump/citm_catalog / 4,ns,1.92608e+06,416105,4.629
|
||||
CopyAssign/canada,ns,3.68545e+06,2.87633e+06,1.281
|
||||
Unflatten/citm_catalog,ns,9.65545e+06,3.47021e+06,2.782
|
||||
FromBinaryBuffer/bjdata_optimized / twitter,ns,3.14359e+06,967149,3.250
|
||||
FromBinaryBuffer/cbor / floats,ns,2.88404e+07,8.22731e+06,3.505
|
||||
Equal/citm_catalog,ns,264018,254367,1.038
|
||||
FromBinaryShape/containers / cbor,ns,1.38356e+07,3.8445e+06,3.599
|
||||
Flatten/citm_catalog,ns,6.43708e+06,2.96328e+06,2.172
|
||||
FromMsgpack/twitter,ns,9.69832e+06,1.46011e+06,6.642
|
||||
FromBinaryBuffer/bson / citm_catalog,ns,3.86353e+06,1.73584e+06,2.226
|
||||
FromBinaryBuffer/msgpack / canada,ns,8.21458e+06,2.2769e+06,3.608
|
||||
"GetArray<std::array<int, 1000>>",ns,2228.17,630.225,3.536
|
||||
SaxParse/twitter,ns,2.4729e+06,472447,5.234
|
||||
ParseString/canada,ns,1.50552e+07,4.35028e+06,3.461
|
||||
ParseFile/canada,ns,1.79247e+07,6.11532e+06,2.931
|
||||
ParseFile/unsigned_ints,ns,1.22099e+08,4.40082e+07,2.774
|
||||
FromCborChunkedString/10000 chunks,ns,89638.1,81306.2,1.102
|
||||
FromBinaryShape/nested / bson,ns,102178,46148.5,2.214
|
||||
ParseIndented/citm_catalog / 4,ns,7.27956e+06,1.75204e+06,4.155
|
||||
Dump/unsigned_ints / 4,ns,3.15101e+07,1.7373e+07,1.814
|
||||
FromBinaryBuffer/bjdata_optimized / canada,ns,8.52567e+06,2.93164e+06,2.908
|
||||
|
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -3,22 +3,24 @@
|
||||
Generated files:
|
||||
|
||||
- perf_blog_overview.svg: speedup of 3.13.0 over 3.12.0 per workload (Google Benchmark suite)
|
||||
- perf_blog_distribution.svg: how many of the suite's benchmarks got how much faster (data/perf_suite.csv)
|
||||
- perf_blog_timeline_<workload>.svg: speedup over 3.12.0 of eight workloads at every develop commit that changed
|
||||
json.hpp or json_view.hpp since 3.12.0, one panel each
|
||||
- perf_blog_swar.svg: the eight-byte string test on one word of input
|
||||
- perf_blog_cliffs.svg: operations that were quadratic or worse, 3.12.0 against 3.13.0 (log scale)
|
||||
- perf_blog_view.svg: parse throughput of json_document and six other document parsers
|
||||
- perf_blog_view_dump.svg: write throughput of the same engines, each writing a document parsed beforehand
|
||||
- perf_blog_usecases_<platform>.svg: libraries x use cases, time relative to the fastest fully validating library
|
||||
(data/perf_usecases_<platform>.json); perf_blog_tables.py prints the tables of the appendix from the same data
|
||||
|
||||
The numbers are copied from the tables of the post (the timeline from data/perf_timeline.csv, which
|
||||
`tests/benchmarks/history/history.py summarize` writes); the tables stay on the page as the accessible fallback. Update both together.
|
||||
Like the figures of charts.py, the SVGs are not images: the page includes their text with
|
||||
`--8<-- "images/<name>.svg"`, so they follow the colours and fonts of the page (`nj-charts` rules in
|
||||
`--8<-- "images/<name>.svg"`, so they follow the colors and fonts of the page (`nj-charts` rules in
|
||||
docs/css/fig-charts.css, `nj-fig` rules in docs/css/brand.css). Run `python3 scripts/figures/perf_blog.py` from
|
||||
docs/mkdocs after changing data or drawing, and commit the generated files.
|
||||
"""
|
||||
import csv
|
||||
import html
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -29,7 +31,8 @@ MINUS = "−"
|
||||
TIMES = "×"
|
||||
|
||||
# words the abbreviation extension (includes/glossary.md) would wrap in <abbr> inside the SVG text
|
||||
ABBR = re.compile(r"(?<![\w;])(BJData|BON8|BSON|CBOR|GCC|UBJSON|MsgPack|JSON|DOM|SAX|SIMD|UTF|ASCII)(?!\w)")
|
||||
GLOSSARY = re.findall(r"^\*\[([^\]]+)\]", (HERE.parents[1] / "includes" / "glossary.md").read_text(), re.M)
|
||||
ABBR = re.compile(r"(?<![\w;])(%s)(?!\w)" % "|".join(sorted(map(re.escape, GLOSSARY), key=len, reverse=True)))
|
||||
|
||||
|
||||
def protect(text):
|
||||
@@ -140,7 +143,7 @@ def overview():
|
||||
hi = max(v for _, _, v in OVERVIEW)
|
||||
hi = math.ceil(hi + 0.5)
|
||||
X = lambda v: L + v / hi * (R - L)
|
||||
pitch, bh, top = 24, 15, 34
|
||||
pitch, bh, top = 24, 15, 8
|
||||
b, desc = [], []
|
||||
y = top
|
||||
group = None
|
||||
@@ -165,23 +168,66 @@ def overview():
|
||||
desc.append("%s %s" % (label, fx(v).replace(TIMES, "x")))
|
||||
b.append('<line class="nj-charts-ref" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (X(1), top, X(1), axis_y))
|
||||
b.append('<line class="nj-charts-zero" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (X(0), top, X(0), axis_y))
|
||||
b.append('<text class="nj-charts-note" x="%g" y="%g">speedup over 3.12.0, more is better (the line at 1%s is 3.12.0)</text>' % (L, 18, TIMES))
|
||||
write("perf_blog_overview.svg", W, axis_y + 28, "nj-charts--blog", "nj-blog-overview",
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">speedup over 3.12.0 (3.12.0 = 1%s), more is better</text>'
|
||||
% ((L + R) / 2, axis_y + 34, TIMES))
|
||||
write("perf_blog_overview.svg", W, axis_y + 44, "nj-charts--blog", "nj-blog-overview",
|
||||
"Speedup of 3.13.0 over 3.12.0 per workload, Apple M1 Max, Apple clang",
|
||||
"Horizontal bars of the speedup, more is better. " + "; ".join(desc) + ".", b)
|
||||
|
||||
|
||||
# ---------- distribution: how many of the suite's benchmarks got how much faster ----------
|
||||
# bins of the speedup (3.12.0 time / 3.13.0 time) over all rows of data/perf_suite.csv
|
||||
BINS = [(0, 1, "below 1" + TIMES), (1, 1.5, "1 to 1.5" + TIMES), (1.5, 2, "1.5 to 2" + TIMES),
|
||||
(2, 3, "2 to 3" + TIMES), (3, 5, "3 to 5" + TIMES), (5, 10, "5 to 10" + TIMES), (10, 1e9, "10" + TIMES + " or more")]
|
||||
|
||||
|
||||
def distribution():
|
||||
rows = list(csv.DictReader((HERE / "data" / "perf_suite.csv").open()))
|
||||
s = sorted(float(r["speedup"]) for r in rows)
|
||||
n = len(s)
|
||||
counts = [sum(lo <= v < hi for v in s) for lo, hi, _ in BINS]
|
||||
geo = math.exp(sum(math.log(v) for v in s) / n)
|
||||
med = (s[(n - 1) // 2] + s[n // 2]) / 2
|
||||
W, H = 560, 250
|
||||
L, R, T, B = 52, 548, 16, 196
|
||||
hi = 10 * math.ceil(max(counts) / 10 + 0.4)
|
||||
Y = lambda c: B - c / hi * (B - T)
|
||||
slot = (R - L) / len(BINS)
|
||||
b, desc = [], []
|
||||
for t in range(0, hi + 1, 20):
|
||||
b.append('<line class="nj-charts-grid" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (L, Y(t), R, Y(t)))
|
||||
b.append('<text class="nj-charts-tick nj-charts-tick--r" x="%g" y="%g">%d</text>' % (L - 6, Y(t) + 4, t))
|
||||
for k, ((lo, _, label), c) in enumerate(zip(BINS, counts)):
|
||||
x = L + k * slot + 8
|
||||
w = slot - 16
|
||||
cls = "nj-charts-c2" if lo < 1 else "nj-charts-c1"
|
||||
b.append('<rect class="%s" x="%.1f" y="%.1f" width="%.1f" height="%.1f" rx="3"><title>%s: %d</title></rect>'
|
||||
% (cls, x, Y(c), w, B - Y(c), label, c))
|
||||
b.append('<text class="nj-charts-val" x="%.1f" y="%.1f" text-anchor="middle">%d</text>' % (x + w / 2, Y(c) - 5, c))
|
||||
b.append('<text class="nj-charts-tick" x="%.1f" y="%.1f">%s</text>' % (x + w / 2, B + 16, label))
|
||||
desc.append("%s: %d" % (label.replace(TIMES, "x"), c))
|
||||
b.append('<line class="nj-charts-axis" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (L, B, R, B))
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">speedup over 3.12.0, more is better</text>' % ((L + R) / 2, H - 10))
|
||||
b.append('<text class="nj-charts-tick" transform="rotate(-90)" x="%g" y="14">number of benchmarks</text>' % (-(T + B) / 2))
|
||||
print("distribution: median %s, geometric mean %s" % (fx(med), fx(geo)))
|
||||
write("perf_blog_distribution.svg", W, H, "nj-charts--blog", "nj-blog-dist",
|
||||
"Speedup of 3.13.0 over 3.12.0 for all %d benchmarks of the suite" % n,
|
||||
"Bars counting the benchmarks per speedup range: " + "; ".join(desc)
|
||||
+ ". Median %s, geometric mean %s." % (fx(med).replace(TIMES, "x"), fx(geo).replace(TIMES, "x")), b)
|
||||
|
||||
|
||||
# ---------- timeline: one panel per workload ----------
|
||||
# (column in perf_timeline.csv, panel title, [PR numbers to label])
|
||||
# (column in perf_timeline.csv, panel title, [(PR, label)]): each labeled step shows the number of the theme that
|
||||
# explains it ("U" for the UTF-8 check of the binary readers that came and went); the appendix maps steps to PRs
|
||||
TIMELINE = [
|
||||
("parse_citm", "parse() citm_catalog.json", [5234, 5283, 5592, 5809, 5810, 5882]),
|
||||
("parse_canada", "parse() canada.json", [5234, 5283, 5617, 5738, 5851, 5882]),
|
||||
("accept_twitter", "accept() twitter.json", [5234, 5283, 5592, 5809, 5810, 5853]),
|
||||
("parse_twitter_ifstream", "parse() twitter.json, std::ifstream", [5828, 5884]),
|
||||
("dump_twitter", "dump() twitter.json", [5285, 5738, 5878, 5883]),
|
||||
("dump_canada", "dump() canada.json", [5285, 5634]),
|
||||
("from_cbor_twitter", "from_cbor() twitter", [5233, 5501, 5185, 5746]),
|
||||
("to_cbor_twitter", "to_cbor() twitter", [5328, 5824, 5885]),
|
||||
("parse_citm", "parse() citm_catalog.json", [(5234, "4"), (5283, "1"), (5592, "4"), (5809, "4"), (5810, "2"), (5882, "4")]),
|
||||
("parse_canada", "parse() canada.json", [(5234, "4"), (5283, "1"), (5617, "3"), (5738, "3"), (5851, "3"), (5882, "4")]),
|
||||
("accept_twitter", "accept() twitter.json", [(5234, "4"), (5283, "1"), (5592, "4"), (5809, "4"), (5810, "2"), (5853, "2")]),
|
||||
("parse_twitter_ifstream", "parse() twitter.json, std::ifstream", [(5828, "1"), (5884, "1")]),
|
||||
("dump_twitter", "dump() twitter.json", [(5285, "6"), (5738, "2"), (5878, "2"), (5883, "2")]),
|
||||
("dump_canada", "dump() canada.json", [(5285, "6"), (5634, "3")]),
|
||||
("from_cbor_twitter", "from_cbor() twitter", [(5233, "1"), (5501, "4"), (5185, "U"), (5746, "U")]),
|
||||
("to_cbor_twitter", "to_cbor() twitter", [(5328, "4"), (5824, "6"), (5885, "6")]),
|
||||
]
|
||||
MONTHS = [("2025-04", "3.12.0"), ("2026-09", "Sep 2026"), ("2026-10", "Oct")]
|
||||
|
||||
@@ -190,7 +236,7 @@ def timeline():
|
||||
rows = list(csv.DictReader((HERE / "data" / "perf_timeline.csv").open()))
|
||||
n = len(rows)
|
||||
W, H = 340, 270
|
||||
L, R, T, B = 44, 330, 62, 208
|
||||
L, R, T, B = 44, 330, 66, 208
|
||||
for col, title, labels in TIMELINE:
|
||||
vals = [float(r[col]) if r[col] else None for r in rows]
|
||||
base = vals[0]
|
||||
@@ -227,35 +273,38 @@ def timeline():
|
||||
# labels for named steps: a dot at the commit and its PR number above it; PRs a few commits apart share a
|
||||
# label, and a label that would overlap one already placed moves up
|
||||
groups = []
|
||||
for pr in labels:
|
||||
for pr, lab in labels:
|
||||
i = next(k for k, r in enumerate(rows) if r["pr"] == str(pr))
|
||||
if groups and X(i) - X(groups[-1][-1][1]) < 6:
|
||||
groups[-1].append((pr, i))
|
||||
groups[-1].append((pr, i, lab))
|
||||
else:
|
||||
groups.append([(pr, i)])
|
||||
groups.append([(pr, i, lab)])
|
||||
# labels in two rows above the plot, each with a guide down to its dot
|
||||
placed = []
|
||||
for g in groups:
|
||||
i = g[-1][1]
|
||||
x, y = X(i), Y(rel[i])
|
||||
for pr, k in g:
|
||||
for pr, k, lab in g:
|
||||
b.append('<circle class="nj-charts-json-dot" cx="%.1f" cy="%.1f" r="3.5"><title>#%s: %s</title></circle>'
|
||||
% (X(k), Y(rel[k]), pr, fx(rel[k])))
|
||||
text = ", ".join("#%s" % pr for pr, _ in g)
|
||||
w = 7.2 * len(text)
|
||||
marks = list(dict.fromkeys(lab for _, _, lab in g))
|
||||
w = 18 * len(marks) - 3
|
||||
tx = min(max(x, L + w / 2), R - w / 2)
|
||||
row = 0
|
||||
while any(r == row and abs(tx - px) < (w + pw) / 2 + 4 for px, r, pw in placed):
|
||||
row += 1
|
||||
placed.append((tx, row, w))
|
||||
ty = 50 - 13 * row
|
||||
b.append('<line class="nj-charts-grid" stroke-dasharray="2 2" x1="%.1f" y1="%.1f" x2="%.1f" y2="%.1f"/>' % (x, ty + 4, x, y - 5))
|
||||
b.append('<text class="nj-charts-note" x="%.1f" y="%.1f" text-anchor="middle">%s</text>' % (tx, ty, text))
|
||||
ty = 46 - 18 * row # the center of the badges
|
||||
b.append('<line class="nj-charts-grid" stroke-dasharray="2 2" x1="%.1f" y1="%.1f" x2="%.1f" y2="%.1f"/>' % (x, ty + 8, x, y - 5))
|
||||
for k, mark in enumerate(marks):
|
||||
cx = tx - w / 2 + 7.5 + 18 * k
|
||||
b.append('<circle class="nj-theme-dot%s" cx="%.1f" cy="%.1f" r="7.5"/>' % ("" if mark.isdigit() else " nj-theme-dot--other", cx, ty))
|
||||
b.append('<text class="nj-theme-num" x="%.1f" y="%.1f" text-anchor="middle">%s</text>' % (cx, ty + 4, mark))
|
||||
b.append('<line class="nj-charts-axis" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (L, B, R, B))
|
||||
b.append('<text class="nj-charts-note" x="%g" y="%g">speedup over 3.12.0 (log scale), more is better</text>' % (8, H - 8))
|
||||
final = rel[-1]
|
||||
desc = ("%s: speedup over 3.12.0 at each of %d commits, ending at %s. Labeled steps: %s."
|
||||
% (title, n, fx(final).replace(TIMES, "x"), ", ".join("#%s" % p for p in labels)))
|
||||
desc = ("%s: speedup over 3.12.0 at each of %d commits, ending at %s. Labeled steps, by theme: %s."
|
||||
% (title, n, fx(final).replace(TIMES, "x"), ", ".join(lab for _, lab in labels)))
|
||||
write("perf_blog_timeline_%s.svg" % col, W, H, "nj-charts--timeline", "nj-blog-tl-" + col,
|
||||
"%s: speedup over 3.12.0 by commit" % title, desc, b)
|
||||
|
||||
@@ -266,8 +315,8 @@ CLIFFS = [
|
||||
("diff() 64k numbers vs []", 4102, 9.98),
|
||||
("diff() 20k objects vs half", 119, 3.85),
|
||||
("parse() callback, 32k objects", 1334, 11.5),
|
||||
("< nested 24 deep", 62.6, 0.002),
|
||||
("< citm, 10x", 274, 3.81),
|
||||
("operator< nested 24 deep", 62.6, 0.002),
|
||||
("operator< citm, 10x", 274, 3.81),
|
||||
("flatten() citm, ordered_json", 253, 1.37),
|
||||
("to_bson() 200 deep, 100x", 19.5, 0.566),
|
||||
("push_back() 100k, diagnostics", 1514, 4.98),
|
||||
@@ -306,84 +355,282 @@ def cliffs():
|
||||
"Dot plot on a logarithmic time axis, less is better; an arrow leads from each 3.12.0 time to the 3.13.0 time. " + "; ".join(desc) + ".", b)
|
||||
|
||||
|
||||
# ---------- json_view: parse throughput ----------
|
||||
# engines and GB/s per file, copied from the table in "Build a different data structure"
|
||||
VIEW_FILES = ["twitter.json", "citm_catalog.json", "canada.json", "jeopardy.json"]
|
||||
VIEW_BYTES = [631514, 1727204, 2251051, 55554625]
|
||||
VIEW_US = [ # (engine, css class, [µs per file]), fresh documents, from the table in "How it compares"
|
||||
("nlohmann/json json_document", "nj-charts-c1", [183, 364, 943, 22681]),
|
||||
("yyjson", "nj-charts-c2", [194, 478, 1489, 30342]),
|
||||
("simdjson DOM", "nj-charts-c3", [202, 486, 2097, 26904]),
|
||||
("sonic-rs (Rust)", "nj-charts-c4", [302, 664, 1907, 30202]),
|
||||
("sonic-cpp", "nj-charts-c5", [379, 986, 2580, 40804]),
|
||||
("Boost.JSON", "nj-charts-c6", [868, 1483, 2481, 88925]),
|
||||
("nlohmann/json json::parse", "nj-charts-c1 nj-charts-dim", [1352, 2386, 5915, 163750]),
|
||||
# ---------- the price: compile time of two programs against 3.12.0 ----------
|
||||
# CPU time of the compiler in ms, median of 9 interleaved rounds, C++20, M1 Max (the post's appendix G)
|
||||
# (3.12.0, 3.13.0, 3.13.0 with a precompiled header, 3.13.0 as a module)
|
||||
PRICE = [
|
||||
("Apple clang 21, -O0 (debug)", [("count_numbers", 959.1, 1155.3, 679.1, 803.9), ("server_config", 1108.2, 1445.1, 763.1, 1057.0)]),
|
||||
("Apple clang 21, -O2", [("count_numbers", 1268.8, 1867.7, 1364.3, 1512.9), ("server_config", 1576.9, 2353.6, 1749.4, 2141.4)]),
|
||||
("GCC 16, -O0 (debug)", [("count_numbers", 1443.7, 2346.4, 1579.2, 1662.5), ("server_config", 1984.8, 2781.7, 2113.3, 2181.1)]),
|
||||
("GCC 16, -O2", [("count_numbers", 2326.9, 3453.3, 2972.4, 2895.3), ("server_config", 2888.7, 4388.6, 3941.8, 3825.5)]),
|
||||
]
|
||||
VIEW = [(e, c, [n / (t * 1000) for n, t in zip(VIEW_BYTES, us)]) for e, c, us in VIEW_US] # GB/s
|
||||
# dump: the bytes of the compact output, and the same engines, from the dump table in "How it compares"
|
||||
DUMP_BYTES = [466906, 500299, 2090234, 52508728]
|
||||
DUMP_US = [
|
||||
("nlohmann/json json_document", "nj-charts-c1", [74.5, 132.8, 1625.1, 14757.2]),
|
||||
("yyjson", "nj-charts-c2", [104.2, 165.7, 1715.7, 20133.5]),
|
||||
("simdjson DOM", "nj-charts-c3", [354.2, 498.2, 6816.1, 54833.4]),
|
||||
("sonic-rs (Rust)", "nj-charts-c4", [212.6, 443.9, 2416.8, 33818.5]),
|
||||
("sonic-cpp", "nj-charts-c5", [104.6, 222.3, 3044.2, 19836.1]),
|
||||
("Boost.JSON", "nj-charts-c6", [497.6, 501.4, 2941.8, 63392.3]),
|
||||
("nlohmann/json json::dump", "nj-charts-c1 nj-charts-dim", [216.7, 285.5, 2244.0, 31205.3]),
|
||||
]
|
||||
DUMP = [(e, c, [n / (t * 1000) for n, t in zip(DUMP_BYTES, us)]) for e, c, us in DUMP_US] # GB/s
|
||||
PRICE_SERIES = [("3.13.0", "nj-charts-c2"), ("3.13.0, precompiled header", "nj-charts-c1"), ("3.13.0, module", "nj-charts-c3")]
|
||||
|
||||
|
||||
def view(VIEW=VIEW, name="perf_blog_view.svg", tid="nj-blog-view",
|
||||
title="Parse throughput of nlohmann/json's json_document and json::parse and five other document parsers, fresh documents, Apple M1 Max",
|
||||
what="parse"):
|
||||
W, L, R = 580, 142, 545
|
||||
hi = max(max(v) for _, _, v in VIEW)
|
||||
hi = math.ceil(hi * 2) / 2
|
||||
X = lambda v: L + v / hi * (R - L)
|
||||
bh, gap = 9, 2
|
||||
group_h = len(VIEW) * (bh + gap) + 14
|
||||
def price():
|
||||
W, L, R = 600, 150, 560
|
||||
lo, hi = -50, 70
|
||||
X = lambda pct: L + (pct - lo) / (hi - lo) * (R - L)
|
||||
pct = lambda new, old: round((new / old - 1) * 100)
|
||||
lab = lambda p: (MINUS + "%d %%" % -p) if p < 0 else "+%d %%" % p
|
||||
bh, pitch, gap = 11, 14, 10
|
||||
b, desc = [], []
|
||||
x, ly = 8, 10
|
||||
for j, (eng, cls, _) in enumerate(VIEW):
|
||||
w = 18 + 7.4 * len(eng) + 16
|
||||
if x + w > W:
|
||||
x, ly = 8, ly + 20
|
||||
b.append('<rect class="%s" x="%g" y="%g" width="12" height="12" rx="3"/>' % (cls, x, ly))
|
||||
b.append('<text class="nj-charts-leg" x="%g" y="%g">%s</text>' % (x + 18, ly + 10.5, esc(eng)))
|
||||
x += w
|
||||
top = ly + 34 # below the last row of the legend
|
||||
axis_y = top + group_h * len(VIEW_FILES)
|
||||
t = 0.5
|
||||
while t <= hi + 1e-9:
|
||||
b.append('<line class="nj-charts-grid" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (X(t), top - 4, X(t), axis_y))
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">%g</text>' % (X(t), axis_y + 16, t))
|
||||
t += 0.5
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">GB/s, more is better</text>' % ((L + R) / 2, axis_y + 34))
|
||||
for i, f in enumerate(VIEW_FILES):
|
||||
y0 = top + i * group_h
|
||||
b.append('<text class="nj-charts-rowlabel" x="%g" y="%g" text-anchor="end">%s</text>' % (L - 8, y0 + group_h / 2, esc(f)))
|
||||
for j, (eng, cls, vals) in enumerate(VIEW):
|
||||
y = y0 + j * (bh + gap)
|
||||
b.append('<path class="%s" d="%s"><title>%s, %s: %.2f GB/s</title></path>' % (cls, hbar(X(0), X(vals[i]), y, bh), esc(eng), esc(f), vals[i]))
|
||||
b.append('<text class="nj-charts-val" x="%g" y="%g">%.2f</text>' % (X(vals[i]) + 4, y + bh - 1.5, vals[i]))
|
||||
desc.append("%s %s %.2f GB/s" % (f, eng, vals[i]))
|
||||
b.append('<line class="nj-charts-zero" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (X(0), top - 4, X(0), axis_y))
|
||||
write(name, W, axis_y + 44, "nj-charts--blog", tid, title,
|
||||
"Grouped horizontal bars of %s throughput in GB/s, more is better. " % what + "; ".join(desc) + ".", b)
|
||||
# the key above the plot
|
||||
x = 8
|
||||
for name, cls in PRICE_SERIES:
|
||||
b.append('<rect class="%s" x="%g" y="6" width="12" height="12" rx="3"/>' % (cls, x))
|
||||
b.append('<text class="nj-charts-leg" x="%g" y="16.5">%s</text>' % (x + 18, esc(name)))
|
||||
x += 18 + 8.2 * len(name) + 22
|
||||
top = 54
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">as fast as 3.12.0</text>' % (X(0), top - 8))
|
||||
y = top
|
||||
for build, progs in PRICE:
|
||||
b.append('<text class="nj-charts-panel" x="8" y="%g">%s</text>' % (y + 14, esc(build)))
|
||||
y += 24
|
||||
y0, at = y, len(b)
|
||||
for prog, old, *new in progs:
|
||||
b.append('<text class="nj-charts-rowlabel nj-charts-code" x="%g" y="%g" text-anchor="end">%s</text>'
|
||||
% (L - 8, y + 1.5 * pitch + 2, prog))
|
||||
for (name, cls), t in zip(PRICE_SERIES, new):
|
||||
p = pct(t, old)
|
||||
b.append('<path class="%s" d="%s"><title>%s, %s, %s: %s ms against %s ms with 3.12.0 (%s)</title></path>'
|
||||
% (cls, hbar(X(0), X(p), y, bh), prog, esc(build), esc(name), format(round(t), ","), format(round(old), ","), lab(p)))
|
||||
if p < 0:
|
||||
b.append('<text class="nj-charts-val" x="%g" y="%g" text-anchor="end">%s</text>' % (X(p) - 5, y + bh - 1.5, lab(p)))
|
||||
else:
|
||||
b.append('<text class="nj-charts-val" x="%g" y="%g">%s</text>' % (X(p) + 5, y + bh - 1.5, lab(p)))
|
||||
desc.append("%s, %s, %s: %s" % (build, prog, name, lab(p).replace(MINUS, "-")))
|
||||
y += pitch
|
||||
y += gap - (pitch - bh)
|
||||
# the grid behind this panel's bars only, so that it doesn't cross the panel titles
|
||||
b[at:at] = ['<line class="nj-charts-grid" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (X(t), y0 - 4, X(t), y - gap + 4)
|
||||
for t in range(lo + 10, hi + 1, 20) if t]
|
||||
y += 8
|
||||
axis_y = y
|
||||
b.append('<line class="nj-charts-ref" x1="%g" y1="%g" x2="%g" y2="%g"/>' % (X(0), top - 2, X(0), axis_y))
|
||||
for t in range(lo + 10, hi + 1, 20):
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">%s</text>' % (X(t), axis_y + 16, "0" if t == 0 else lab(t)))
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">compile time compared with 3.12.0, less is better</text>' % ((L + R) / 2, axis_y + 34))
|
||||
write("perf_blog_price.svg", W, axis_y + 44, "nj-charts--blog", "nj-blog-price",
|
||||
"Compile time of two programs with 3.13.0, alone, with a precompiled header and as a module, compared with 3.12.0",
|
||||
"Horizontal bars of the change in compile time against 3.12.0, less is better. " + "; ".join(desc) + ".", b)
|
||||
|
||||
|
||||
# ---------- use cases: libraries x use cases, relative time ----------
|
||||
# data/perf_usecases_<platform>.json is written by tests/benchmarks/json_view/others/summarize.py --json
|
||||
UC_FILES = ["twitter", "citm_catalog", "canada", "jeopardy"]
|
||||
USE_CASES = [("validate", "validate"), ("sum", "sum"), ("select", "select"),
|
||||
("edit", "edit"), ("minify", "minify"), ("write", "write")]
|
||||
UC_LIBS = { # library in the JSON -> row label
|
||||
"nlohmann json_document": "nlohmann/json json_document", "nlohmann json": "nlohmann/json json",
|
||||
"yyjson": "yyjson", "simdjson": "simdjson", "sonic-cpp": "sonic-cpp", "Glaze": "Glaze", "Boost.JSON": "Boost.JSON",
|
||||
"RapidJSON": "RapidJSON", "sonic-rs": "sonic-rs (Rust)", "simd-json": "simd-json (Rust)", "serde_json": "serde_json (Rust)",
|
||||
}
|
||||
UC_PLATFORMS = [("mac", "Use cases: time of each library relative to the fastest, Apple M1 Max"),
|
||||
("x86-gcc", "Use cases: time of each library relative to the fastest, x86-64 with GCC 16.2"),
|
||||
("x86-gcc13", "Use cases: time of each library relative to the fastest, x86-64 with GCC 13"),
|
||||
("x86-clang", "Use cases: time of each library relative to the fastest, x86-64 with Clang 23")]
|
||||
LEVEL_MARK = {"no UTF-8": "U", "partial": "P", "full": ""}
|
||||
# why a cell is empty (the tooltips of the n/a cells)
|
||||
NA = {("simdjson", "edit"): "simdjson's documents are read-only: it has no API to change a value and write the result"}
|
||||
|
||||
|
||||
def geomean(xs):
|
||||
return math.exp(sum(math.log(x) for x in xs) / len(xs))
|
||||
|
||||
|
||||
def normalize_usecases(data):
|
||||
"""Chooses each library's best variants again with the rules of the post: variants that copy each number's text
|
||||
instead of formatting it (json_document's number_format::source, RapidJSON's raw numbers) and variants that reuse
|
||||
a parser or document across calls are measured but not chosen"""
|
||||
data = json.loads(json.dumps(data))
|
||||
rank = {"full": 0, "n/a": 0, "no UTF-8": 1, "partial": 2}
|
||||
for files in data.values():
|
||||
for cell in files.values():
|
||||
for e in cell.values():
|
||||
vs = [(v["us"], k, v["level"]) for k, v in e["variants"].items()
|
||||
if not any(w in k for w in ("source numbers", "raw numbers", "reused"))]
|
||||
full = [v for v in vs if rank[v[2]] == 0]
|
||||
pick = lambda v: {"us": v[0], "variant": v[1], "level": "full" if rank[v[2]] == 0 else v[2]}
|
||||
e["best"] = pick(min(full)) if full else None
|
||||
e["best_any"] = pick(min(vs))
|
||||
return data
|
||||
|
||||
|
||||
FILE_NAMES = {"twitter": "twitter.json", "citm_catalog": "citm_catalog.json", "canada": "canada.json",
|
||||
"jeopardy": "jeopardy.json"}
|
||||
# what a variant that checks less skips, worded as the trade it is (the tooltips of the heatmaps)
|
||||
CHECKS_LESS = [
|
||||
("simdjson", "On-Demand", "reads lazily: On-Demand reads only what you ask for and steps over the rest by matching brackets"),
|
||||
("simdjson", "minify", "simdjson::minify() only removes whitespace, without parsing the values"),
|
||||
("Glaze", "lazy_json", "reads lazily: lazy_json reads only what you ask for and steps over the rest"),
|
||||
("Glaze", "minify_json", "minify_json only removes whitespace, without parsing the values"),
|
||||
("Glaze", "structs", "reads into structs and steps over fields it doesn't know without checking them "
|
||||
"(validate_skipped checks them too)"),
|
||||
("RapidJSON", "", "trusts UTF-8: RapidJSON's default (kParseValidateEncodingFlag checks it)"),
|
||||
("sonic-cpp", "", "trusts UTF-8: sonic-cpp 1.0.2 doesn't check it"),
|
||||
("serde_json", "", "trusts UTF-8 in the strings it skips (from_slice); strings it reads are checked"),
|
||||
]
|
||||
|
||||
|
||||
def checks_less(lib, variant):
|
||||
for l, v, text in CHECKS_LESS:
|
||||
if l == lib and v in variant:
|
||||
return text
|
||||
return "checks less"
|
||||
|
||||
|
||||
def usecase_table(data):
|
||||
"""{library: {use case: cell}}: per library the geometric mean over UC_FILES of its fastest fully validating time
|
||||
("full") and of its fastest time at any level ("alt", only if it is at least 3 % faster), each divided by the best
|
||||
such fully validating mean in the use case, so that every file counts equally and the fastest library is 1.00"""
|
||||
out = {}
|
||||
data = normalize_usecases(data)
|
||||
for uc, _ in USE_CASES:
|
||||
files = data.get(uc, {})
|
||||
if not files:
|
||||
continue
|
||||
cells = {}
|
||||
for lib in UC_LIBS:
|
||||
es = [files.get(f, {}).get(lib) for f in UC_FILES]
|
||||
if not all(es):
|
||||
continue
|
||||
full = [e["best"] for e in es]
|
||||
alt = [e["best_any"] for e in es]
|
||||
cells[lib] = {"full": geomean([b["us"] for b in full]) if all(full) else None, "alt": geomean([a["us"] for a in alt]),
|
||||
"files": [(f, b, a) for f, b, a in zip(UC_FILES, full, alt)]}
|
||||
ref = min(c["full"] for c in cells.values() if c["full"])
|
||||
for lib, c in cells.items():
|
||||
full = c["full"] / ref if c["full"] else None
|
||||
alt = c["alt"] / ref
|
||||
if full is not None and alt > full * 0.97:
|
||||
alt = None
|
||||
levels = {a["level"] for _, _, a in c["files"]} if alt is not None else set()
|
||||
level = "partial" if "partial" in levels else "no UTF-8" if "no UTF-8" in levels else "full"
|
||||
out.setdefault(lib, {})[uc] = {"full": full, "alt": alt, "level": level, "files": c["files"]}
|
||||
return out
|
||||
|
||||
|
||||
# cell colors: green for the fastest, gray around twice the time, red for the slowest; the upper bounds of the steps
|
||||
HEAT_STEPS = [1.15, 1.5, 2.5, 5, float("inf")]
|
||||
HEAT_LABELS = ["below 1.15", "1.15 to 1.5", "1.5 to 2.5", "2.5 to 5", "5 and more"]
|
||||
|
||||
|
||||
def heat(r):
|
||||
return next(i for i, b in enumerate(HEAT_STEPS) if r < b)
|
||||
|
||||
|
||||
def fr(r):
|
||||
return "%.2f" % r if r < 9.995 else "%.1f" % r if r < 99.95 else "%.0f" % r
|
||||
|
||||
|
||||
def fus(us):
|
||||
return "{:,.0f} µs".format(us) if us >= 100 else "%.1f µs" % us
|
||||
|
||||
|
||||
def tip(lib, uc_label, c, half):
|
||||
"""the tooltip of a cell: lines separated by newlines ( in the attribute), the first one the heading"""
|
||||
if half == "full":
|
||||
lines = ["%s · %s" % (UC_LIBS[lib], uc_label),
|
||||
"%s× the time of the fastest library (fully validating)" % fr(c["full"])]
|
||||
picks = [(f, b) for f, b, _ in c["files"]]
|
||||
else:
|
||||
lines = ["%s · %s, checking less" % (UC_LIBS[lib], uc_label),
|
||||
"%s× the time of the fastest fully validating library" % fr(c["alt"])]
|
||||
picks = [(f, a) for f, _, a in c["files"]]
|
||||
ways = sorted({e["variant"] for _, e in picks})
|
||||
if len(ways) == 1:
|
||||
lines.append("way: %s" % ways[0])
|
||||
for f, e in picks:
|
||||
lines.append("%s: %s%s%s" % (FILE_NAMES[f], fus(e["us"]), "" if len(ways) == 1 else " · " + e["variant"],
|
||||
" (validates fully)" if half == "alt" and e["level"] == "full" else ""))
|
||||
if half == "alt":
|
||||
seen = []
|
||||
for _, e in picks:
|
||||
if e["level"] != "full":
|
||||
t = checks_less(lib, e["variant"])
|
||||
if t not in seen:
|
||||
seen.append(t)
|
||||
lines += seen
|
||||
if any("(copy)" in e["variant"] for _, e in picks):
|
||||
lines.append("(copy): parses in place, on a copy of the input that is part of the time")
|
||||
return " ".join(lines)
|
||||
|
||||
|
||||
def heatmap(platform, title):
|
||||
data = json.load((HERE / "data" / ("perf_usecases_%s.json" % platform)).open())
|
||||
t = usecase_table(data)
|
||||
libs = [l for l in UC_LIBS if l in t]
|
||||
score = lambda l: geomean([v["full"] if v["full"] else v["alt"] * 1.5 for v in t[l].values()])
|
||||
libs.sort(key=score)
|
||||
W, L, cw, ch, top = 784, 224, 92, 36, 44
|
||||
b, desc = [], []
|
||||
for j, (uc, label) in enumerate(USE_CASES):
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">%s</text>' % (L + j * cw + cw / 2, top - 10, esc(label)))
|
||||
for i, lib in enumerate(libs):
|
||||
y = top + i * ch
|
||||
cls = "nj-charts-name-json" if lib.startswith("nlohmann") else ""
|
||||
b.append('<text class="nj-charts-rowlabel %s" x="%g" y="%g" text-anchor="end">%s</text>' % (cls, L - 8, y + ch / 2 + 4, esc(UC_LIBS[lib])))
|
||||
for j, (uc, label) in enumerate(USE_CASES):
|
||||
x = L + j * cw
|
||||
c = t[lib].get(uc)
|
||||
if c is None:
|
||||
b.append('<g class="nj-heat-cell" tabindex="0" data-tip="%s">' % esc("%s · %s %s" % (UC_LIBS[lib], label, NA.get((lib, uc), "the library has no API for this use case"))))
|
||||
b.append('<rect class="nj-heat-na" x="%g" y="%g" width="%g" height="%g" rx="3"/>' % (x + 1, y + 1, cw - 2, ch - 2))
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">n/a</text>' % (x + cw / 2, y + ch / 2 + 4))
|
||||
b.append('</g>')
|
||||
desc.append("%s %s: n/a" % (UC_LIBS[lib], label))
|
||||
continue
|
||||
halves = []
|
||||
if c["full"] is not None:
|
||||
halves.append(("full", c["full"]))
|
||||
if c["alt"] is not None:
|
||||
halves.append(("alt", c["alt"]))
|
||||
hw = (cw - 2) / len(halves)
|
||||
for k, (half, r) in enumerate(halves):
|
||||
hx = x + 1 + k * hw
|
||||
gap = 1 if len(halves) == 2 else 0
|
||||
b.append('<g class="nj-heat-cell" tabindex="0" data-tip="%s">' % esc(tip(lib, label, c, half)))
|
||||
b.append('<rect class="nj-heat-%d" x="%g" y="%g" width="%g" height="%g" rx="3"/>'
|
||||
% (heat(r), hx + (gap if k else 0), y + 1, hw - gap, ch - 2))
|
||||
cx = hx + hw / 2
|
||||
if half == "alt":
|
||||
b.append('<text class="nj-charts-cell" x="%g" y="%g" text-anchor="middle">%s</text>' % (cx, y + 16, fr(r)))
|
||||
b.append('<text class="nj-charts-cell nj-charts-cell--small" x="%g" y="%g" text-anchor="middle">%s</text>'
|
||||
% (cx, y + 29, LEVEL_MARK[c["level"]]))
|
||||
else:
|
||||
b.append('<text class="nj-charts-cell" x="%g" y="%g" text-anchor="middle">%s</text>' % (cx, y + ch / 2 + 4, fr(r)))
|
||||
b.append('</g>')
|
||||
if c["full"] is None: # no fully validating way: the left half stays empty
|
||||
pass
|
||||
desc.append("%s %s: %s%s" % (UC_LIBS[lib], label, fr(c["full"]) if c["full"] else "no fully validating way",
|
||||
(" (%s checking less, %s)" % (fr(c["alt"]), c["level"])) if c["alt"] else ""))
|
||||
# the key, set apart from the grid and narrower than its columns, so that it doesn't read as a row of it
|
||||
yb = top + len(libs) * ch + 28
|
||||
b.append('<text class="nj-charts-tick nj-charts-tick--l" x="8" y="%g">time relative to the fastest fully validating library, less is better</text>' % (yb + 4))
|
||||
kw, ky = 76, yb + 14
|
||||
for i, lab in enumerate(HEAT_LABELS):
|
||||
# each swatch in its own group, so that the ink rule for the strong steps (a sibling selector) stays inside it
|
||||
b.append('<g><rect class="nj-heat-%d" x="%g" y="%g" width="%g" height="20" rx="3"/>' % (i, 8 + i * (kw + 4), ky, kw))
|
||||
b.append('<text class="nj-charts-cell nj-charts-cell--small" x="%g" y="%g" text-anchor="middle">%s</text></g>' % (8 + i * (kw + 4) + kw / 2, ky + 14, esc(lab)))
|
||||
nx = 8 + len(HEAT_LABELS) * (kw + 4) + 12
|
||||
b.append('<rect class="nj-heat-na" x="%g" y="%g" width="%g" height="20" rx="3"/>' % (nx, ky, 40))
|
||||
b.append('<text class="nj-charts-tick" x="%g" y="%g">n/a</text>' % (nx + 20, ky + 14))
|
||||
b.append('<text class="nj-charts-tick nj-charts-tick--l" x="%g" y="%g">no API for it</text>' % (nx + 48, ky + 14))
|
||||
write("perf_blog_usecases_%s.svg" % platform, W, ky + 28, "nj-charts--blog nj-charts--heat", "nj-blog-uc-" + platform, title,
|
||||
"Table of relative times, libraries by use case. " + "; ".join(desc) + ".", b)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
swar()
|
||||
if OVERVIEW:
|
||||
overview()
|
||||
distribution()
|
||||
if TIMELINE:
|
||||
timeline()
|
||||
if CLIFFS:
|
||||
cliffs()
|
||||
if VIEW:
|
||||
view()
|
||||
if DUMP:
|
||||
view(DUMP, "perf_blog_view_dump.svg", "nj-blog-view-dump",
|
||||
"Write throughput of nlohmann/json's json_document and json, and five other libraries, each writing a "
|
||||
"document parsed beforehand, Apple M1 Max", "write")
|
||||
price()
|
||||
for platform, title in UC_PLATFORMS:
|
||||
if (HERE / "data" / ("perf_usecases_%s.json" % platform)).exists():
|
||||
heatmap(platform, title)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Prints the tables of Appendix E of home/blog/how_3_13_0_got_faster.md (comparison with other libraries) as Markdown,
|
||||
from data/perf_usecases_<platform>.json, which tests/benchmarks/json_view/others/summarize.py --json writes.
|
||||
|
||||
Usage: python3 scripts/figures/perf_blog_tables.py <platform> [title]
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from perf_blog import normalize_usecases
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
FILES = [("twitter", "`twitter.json`"), ("citm_catalog", "`citm_catalog.json`"), ("canada", "`canada.json`"),
|
||||
("jeopardy", "`jeopardy.json`"), ("status", "one status (2.5 KB)")]
|
||||
USE_CASES = [("validate", "validate"), ("sum", "sum"), ("select", "select"), ("edit", "edit"), ("minify", "minify"),
|
||||
("write", "write")]
|
||||
NAMES = {"nlohmann json_document": "nlohmann/json `json_document`", "nlohmann json": "nlohmann/json `json`",
|
||||
"sonic-rs": "sonic-rs (Rust)", "simd-json": "simd-json (Rust)", "serde_json": "serde_json (Rust)"}
|
||||
MARK = {"no UTF-8": "U", "partial": "P"}
|
||||
|
||||
|
||||
def fmt(us):
|
||||
if us >= 10000:
|
||||
return "{:,.0f}".format(us)
|
||||
if us >= 100:
|
||||
return "{:,.0f}".format(us)
|
||||
return "%.1f" % us
|
||||
|
||||
|
||||
def table(data, uc, indent):
|
||||
files = [f for f, _ in FILES if f in data.get(uc, {})]
|
||||
libs = list(data[uc][files[0]].keys())
|
||||
jd = lambda f: data[uc][f]["nlohmann json_document"]["best"]["us"]
|
||||
libs.sort(key=lambda l: sum(((data[uc][f][l]["best"] or data[uc][f][l]["best_any"])["us"] / jd(f)) for f in files))
|
||||
head = "| library | variant | " + " | ".join(n for f, n in FILES if f in files) + " |"
|
||||
rows = [head, "| --- | --- | " + " | ".join("--:" for _ in files) + " |"]
|
||||
for lib in libs:
|
||||
best = [data[uc][f][lib]["best"] for f in files]
|
||||
anyv = [data[uc][f][lib]["best_any"] for f in files]
|
||||
names = sorted({b["variant"] for b in best if b} | {a["variant"] + " [%s]" % MARK[a["level"]] for b, a in zip(best, anyv) if a["level"] in MARK and (not b or a["us"] < b["us"])})
|
||||
cells = []
|
||||
for b, a in zip(best, anyv):
|
||||
c = fmt(b["us"]) if b else "–"
|
||||
if a["level"] in MARK and (not b or a["us"] < b["us"] * 0.97):
|
||||
c += " (%s %s)" % (MARK[a["level"]], fmt(a["us"]))
|
||||
cells.append(c)
|
||||
bold = "**%s**" if lib == "nlohmann json_document" else "%s"
|
||||
rows.append("| %s | %s | %s |" % (bold % NAMES.get(lib, lib), ", ".join(names), " | ".join(cells)))
|
||||
return "\n".join(indent + r for r in rows)
|
||||
|
||||
|
||||
def main():
|
||||
platform = sys.argv[1]
|
||||
data = json.load((HERE / "data" / ("perf_usecases_%s.json" % platform)).open())
|
||||
data = normalize_usecases(data) # the same choices as the figure
|
||||
out = []
|
||||
for uc, label in USE_CASES:
|
||||
if uc not in data:
|
||||
continue
|
||||
out.append("**%s**\n" % label)
|
||||
out.append(table(data, uc, ""))
|
||||
out.append("")
|
||||
print("\n".join(out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user