From 7140ffb43d79735cd91b0b12b80c1f98f0bdaf37 Mon Sep 17 00:00:00 2001 From: Vladimir Dubrovin <3proxy@3proxy.ru> Date: Wed, 14 Oct 2026 11:02:30 +0300 Subject: [PATCH] bench: 90th percentile latency, per-connection link emulation wrk's and mix's 90th percentile is recorded (p90_ms) and shown in the report.py --latency tables when present. --flow-rate limits each connection between the client and the proxy (fq maxrate), under --netem when both are given, now with a large netem queue. Both directions are shaped in the client's namespace, the incoming one through an ifb, so the emulation runs on the client's CPUs; segmentation offload and GRO are off on that link so that netem drops single packets. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 14 ++++++++++++-- bench/bench.py | 48 ++++++++++++++++++++++++++++++++++++++--------- bench/mix/main.go | 4 +++- bench/report.py | 19 +++++++++++-------- 4 files changed, 65 insertions(+), 20 deletions(-) diff --git a/bench/README.md b/bench/README.md index ca03c81..a0bbf83 100644 --- a/bench/README.md +++ b/bench/README.md @@ -232,8 +232,18 @@ cluster by default, are lifted for the scenarios that open more. The official release binaries run as they are (`envoy` on the PATH). `--scenario` picks scenarios, `--reps` and `--duration` set the repetitions -and their length, `--netem "delay 1ms rate 10gbit"` puts a slower, longer link -between the client and the proxy, `--mtu` changes the links' MTU. +and their length, `--mtu` changes the links' MTU. Two options make the link +between the client and the proxy more like the clients' links of a real +proxy, both ways: `--netem "delay 5ms loss 0.5%"` adds netem's delay and +loss (`rate` there would be the link's as a whole), `--flow-rate 20mbit` +limits each connection (fq `maxrate`). Both are applied in the client's +namespace, on its way out and on an ifb its way in is redirected through, +so the emulation's work is done on the client's CPUs and not charged to the +proxy. Segmentation offload and GRO are turned off on that link, so that +netem drops single packets: the proxy then sends and receives packets of the +MTU's size, as it would from slow clients, which costs it 2-3 times the CPU +per exchange of a plain veth. Shaped, the client's CPUs set the pace of the +full-load mix scenarios; the fixed-rate `-3k` ones are what compares. ## Caveats diff --git a/bench/bench.py b/bench/bench.py index e8ca288..22f12a7 100644 --- a/bench/bench.py +++ b/bench/bench.py @@ -201,10 +201,37 @@ def net_up(args): ("bprx", "bp1", "prx"), ("bsrv", "bs0", "srv")): sh(f"ip netns exec {ns} sh -c 'echo {cpumask(args.cpus[role])} " f"> /sys/class/net/{dev}/queues/rx-0/rps_cpus'", check=False) - if args.netem: + # The client link, both ways: netem for its delay and loss, fq for a + # rate of each connection, under netem when both are given. Both ways + # are shaped in the client's namespace, so the work is the client's + # CPUs', not charged to the proxy: what the client sends on bc0's way + # out, what it receives on an ifb its way in is redirected through. + # Segmentation offload is off on the link, so that what netem drops is + # one packet rather than a 64 KB burst and fq paces packets the size of + # the MTU. + if args.netem or args.flow_rate: + netem = args.netem or "" + if netem and "limit" not in netem.split(): + # netem's queue holds 1000 packets by default: too few for + # thousands of connections, which would lose more than asked + netem += " limit 1000000" + fq = f"fq maxrate {args.flow_rate} flow_limit 1000" + sh("modprobe ifb numifbs=0", check=False) + sh("ip -n bcli link add bci type ifb") + sh("ip -n bcli link set bci up") + sh("ip netns exec bcli tc qdisc add dev bc0 handle ffff: ingress") + sh("ip netns exec bcli tc filter add dev bc0 parent ffff: matchall " + "action mirred egress redirect dev bci") for ns, dev in (("bcli", "bc0"), ("bprx", "bp0")): - sh(f"ip netns exec {ns} tc qdisc add dev {dev} root netem {args.netem}") - + sh(f"ip netns exec {ns} ethtool -K {dev} gso off tso off gro off", check=False) + for dev in ("bc0", "bci"): + if netem and args.flow_rate: + sh(f"ip netns exec bcli tc qdisc add dev {dev} root handle 1: netem {netem}") + sh(f"ip netns exec bcli tc qdisc add dev {dev} parent 1:1 handle 10: {fq}") + elif netem: + sh(f"ip netns exec bcli tc qdisc add dev {dev} root netem {netem}") + else: + sh(f"ip netns exec bcli tc qdisc add dev {dev} root {fq}") def net_down(): for ns in ("bcli", "bprx", "bsrv"): @@ -959,7 +986,7 @@ def run_http(proxy, params, args, work, mark): res["rate"] = float(m.group(1)) if m else 0.0 m = re.search(r"Transfer/sec:\s+([\d.]+)(\w+)", out) res["gbit"] = float(m.group(1)) * UNITS[m.group(2)] * 8 / 1e9 if m else 0.0 - for pct in ("50", "99"): + for pct in ("50", "90", "99"): m = re.search(rf"^\s+{pct}%\s+(\S+)", out, re.M) res[f"p{pct}_ms"] = to_ms(m.group(1)) if m else float("nan") errors = 0 @@ -1013,7 +1040,7 @@ def run_stream(proxy, params, args, work, mark): finally: if wrapper: stop(wrapper) - res = {"raw": out, "rate": float("nan"), "p50_ms": float("nan"), + res = {"raw": out, "rate": float("nan"), "p50_ms": float("nan"), "p90_ms": float("nan"), "p99_ms": float("nan"), "requests": 0, "errors": 0, "retrans": 0, "timeouts": 0} try: j = json.loads(out) @@ -1087,8 +1114,8 @@ def run_mix(proxy, params, args, work, mark): res["rate"] = j.get("rate", 0.0) res["gbit"] = j.get("gbit", 0.0) res["bytes"] = j.get("bytes", 0) - res["p50_ms"] = j.get("p50_ms", float("nan")) - res["p99_ms"] = j.get("p99_ms", float("nan")) + for pct in ("50", "90", "99"): + res[f"p{pct}_ms"] = j.get(f"p{pct}_ms", float("nan")) res["timeouts"] = j.get("timeouts", 0) # connections of the pool which could not be opened count too res["errors"] = j.get("errors", 0) + max(j.get("pool", 0) - j.get("opened", 0), 0) @@ -1151,7 +1178,7 @@ def measure(proxy, scenario, args, work): # --- main -------------------------------------------------------------------- -FIELDS = ["proxy", "scenario", "rep", "gbit", "rate", "p50_ms", "p99_ms", "errors", "timeouts", "retrans", +FIELDS = ["proxy", "scenario", "rep", "gbit", "rate", "p50_ms", "p90_ms", "p99_ms", "errors", "timeouts", "retrans", "prx_cpu_s", "prx_task_s", "cpu_s_per_gb", "cpu_us_per_req", "peak_mb", "cli_util", "prx_util", "srv_util", "cli_thread", "srv_thread", "bound"] @@ -1188,7 +1215,10 @@ def main(): ap.add_argument("--cpus", default="cli=0-3,prx=4,srv=5-7", help="CPUs per role (default %(default)s)") ap.add_argument("--mtu", type=int, default=1500) - ap.add_argument("--netem", help='netem on the client link, e.g. "delay 1ms rate 10gbit"') + ap.add_argument("--netem", help='netem on the client link, both ways, e.g. ' + '"delay 5ms loss 0.5%%"') + ap.add_argument("--flow-rate", help="the client link's rate for each connection, " + "each way, e.g. 20mbit (fq maxrate)") ap.add_argument("--out", default="bench/results", help="directory for the CSV and logs") ap.add_argument("--mix", help="the bench/mix binary for the mix-* scenarios " "(default: bench/mix/mix-linux-, see bench/mix/build.sh)") diff --git a/bench/mix/main.go b/bench/mix/main.go index dabde36..e09982d 100644 --- a/bench/mix/main.go +++ b/bench/mix/main.go @@ -722,6 +722,7 @@ type summary struct { Gbit float64 `json:"gbit"` Bytes uint64 `json:"bytes"` P50ms float64 `json:"p50_ms"` + P90ms float64 `json:"p90_ms"` P99ms float64 `json:"p99_ms"` FirstP50ms float64 `json:"first_p50_ms"` FirstP99ms float64 `json:"first_p99_ms"` @@ -847,7 +848,8 @@ func runClient(cl *client, active int, rate float64, warmup, duration time.Durat Bytes: cl.bytes.Load(), Errors: cl.errs.Load(), Timeouts: cl.timeouts.Load(), Reconnects: cl.reconnects.Load(), Late: late.Load(), Aborts: cl.aborts.Load(), Resets: cl.resets.Load(), HalfCloses: cl.halfcloses.Load(), - P50ms: cl.complete.quantile(0.5), P99ms: cl.complete.quantile(0.99), + P50ms: cl.complete.quantile(0.5), P90ms: cl.complete.quantile(0.9), + P99ms: cl.complete.quantile(0.99), FirstP50ms: cl.first.quantile(0.5), FirstP99ms: cl.first.quantile(0.99)} s.DialError, _ = cl.dialErr.Load().(string) s.ExchError, _ = cl.exchErr.Load().(string) diff --git a/bench/report.py b/bench/report.py index d6daadc..4b04ad3 100644 --- a/bench/report.py +++ b/bench/report.py @@ -27,9 +27,10 @@ of the group whose range (the average of its slowest runs to that of its fastest) reaches the best one's; * marks the best of the row, both groups together, in the same way. --split puts the splice group in a table of its own (* still compares across both), --prefix connect- takes only the scenarios so -named. --latency makes one table of the runs' 50th and 99th percentile -latency for a single CSV instead, ranked by the 99th. --overall marks the best of the row only, in bold and with *: for -tables where one group has a single proxy, its group's best says nothing. +named. --latency makes one table of the runs' 50th, 90th and 99th +percentile latency for a single CSV instead, ranked by the 99th. --overall +marks the best of the row only, in bold and with *: for tables where one +group has a single proxy, its group's best says nothing. """ import argparse @@ -237,8 +238,8 @@ def combined(machines, labels, splice, prefix=None, split=False, overall_only=Fa def latency(path, labels, splice, prefix=None): - """Latency of one machine's runs: each cell the medians of the runs' 50th - and 99th percentiles, in milliseconds. Bold: the lowest 99th percentile + """Latency of one machine's runs: each cell the medians of the runs' 50th, + 90th (when recorded) and 99th percentiles, in milliseconds. Bold: the lowest 99th percentile of its group and any other whose runs overlap it; *: the same over the row. Scenarios without a latency (iperf3 streams) are left out.""" by, proxies, scenarios = {}, [], [] @@ -269,7 +270,9 @@ def latency(path, labels, splice, prefix=None): rule.append(":-:") head += [labels.get(p, p) for p in g] rule += ["---:"] * len(g) - out = ["| scenario, p50 / p99 ms | " + " | ".join(head) + " |", + pcts = ["p50_ms", "p90_ms", "p99_ms"] if any(values(rs, "p90_ms") for rs in by.values()) \ + else ["p50_ms", "p99_ms"] + out = [f"| scenario, {' / '.join(c[:3] for c in pcts)} ms | " + " | ".join(head) + " |", "|---|" + "|".join(rule) + "|"] for s in scenarios: overall = winners(by, s, everyone, "p99_ms", False) @@ -283,7 +286,7 @@ def latency(path, labels, splice, prefix=None): if not sel: cells.append("–") continue - text = f"{ms(median(sel, 'p50_ms'))} / {ms(median(sel, 'p99_ms'))}" + text = " / ".join(ms(median(sel, c)) for c in pcts) if p in best: text = f"**{text}**" if p in overall: @@ -312,7 +315,7 @@ def main(): ap.add_argument("--splice", action="append", default=[], help="a proxy (as named in the CSV) using splice(); repeat") ap.add_argument("--latency", action="store_true", - help="one table of latency, 50th / 99th percentile, for the " + help="one table of latency, 50th / 90th / 99th percentile, for the " "one CSV given") args = ap.parse_args() labels = dict(l.split("=", 1) for l in args.label)