bughz/3proxy
1
0
Fork 0
mirror of https://github.com/3proxy/3proxy.git synced 2026-10-16 00:25:48 +08:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

bench: 90th percentile latency, per-connection link emulation

Browse Source 
wrk's and mix's 90th percentile is recorded (p90_ms) and shown in the
report.py --latency tables when present.

--flow-rate limits each connection between the client and the proxy
(fq maxrate), under --netem when both are given, now with a large netem
queue. Both directions are shaped in the client's namespace, the
incoming one through an ifb, so the emulation runs on the client's CPUs;
segmentation offload and GRO are off on that link so that netem drops
single packets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vladimir Dubrovin 2026-10-14 11:02:30 +03:00
parent b1a551c5b1
commit 7140ffb43d
4 changed files with 65 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
bench/README.md
View File

@ -232,8 +232,18 @@ cluster by default, are lifted for the scenarios that open more. The
official release binaries run as they are (`envoy` on the PATH).
`--scenario` picks scenarios, `--reps` and `--duration` set the repetitions
and their length, `--netem "delay 1ms rate 10gbit"` puts a slower, longer link
between the client and the proxy, `--mtu` changes the links' MTU.
and their length, `--mtu` changes the links' MTU. Two options make the link
between the client and the proxy more like the clients' links of a real
proxy, both ways: `--netem "delay 5ms loss 0.5%"` adds netem's delay and
loss (`rate` there would be the link's as a whole), `--flow-rate 20mbit`
limits each connection (fq `maxrate`). Both are applied in the client's
namespace, on its way out and on an ifb its way in is redirected through,
so the emulation's work is done on the client's CPUs and not charged to the
proxy. Segmentation offload and GRO are turned off on that link, so that
netem drops single packets: the proxy then sends and receives packets of the
MTU's size, as it would from slow clients, which costs it 2-3 times the CPU
per exchange of a plain veth. Shaped, the client's CPUs set the pace of the
full-load mix scenarios; the fixed-rate `-3k` ones are what compares.
## Caveats

48
bench/bench.py
View File

@ -201,10 +201,37 @@ def net_up(args):
("bprx", "bp1", "prx"), ("bsrv", "bs0", "srv")):
sh(f"ip netns exec {ns} sh -c 'echo {cpumask(args.cpus[role])} "
f"> /sys/class/net/{dev}/queues/rx-0/rps_cpus'", check=False)
if args.netem:
# The client link, both ways: netem for its delay and loss, fq for a
# rate of each connection, under netem when both are given. Both ways
# are shaped in the client's namespace, so the work is the client's
# CPUs', not charged to the proxy: what the client sends on bc0's way
# out, what it receives on an ifb its way in is redirected through.
# Segmentation offload is off on the link, so that what netem drops is
# one packet rather than a 64 KB burst and fq paces packets the size of
# the MTU.
if args.netem or args.flow_rate:
netem = args.netem or ""
if netem and "limit" not in netem.split():
# netem's queue holds 1000 packets by default: too few for
# thousands of connections, which would lose more than asked
netem += " limit 1000000"
fq = f"fq maxrate {args.flow_rate} flow_limit 1000"
sh("modprobe ifb numifbs=0", check=False)
sh("ip -n bcli link add bci type ifb")
sh("ip -n bcli link set bci up")
sh("ip netns exec bcli tc qdisc add dev bc0 handle ffff: ingress")
sh("ip netns exec bcli tc filter add dev bc0 parent ffff: matchall "
"action mirred egress redirect dev bci")
for ns, dev in (("bcli", "bc0"), ("bprx", "bp0")):
sh(f"ip netns exec {ns} tc qdisc add dev {dev} root netem {args.netem}")
sh(f"ip netns exec {ns} ethtool -K {dev} gso off tso off gro off", check=False)
for dev in ("bc0", "bci"):
if netem and args.flow_rate:
sh(f"ip netns exec bcli tc qdisc add dev {dev} root handle 1: netem {netem}")
sh(f"ip netns exec bcli tc qdisc add dev {dev} parent 1:1 handle 10: {fq}")
elif netem:
sh(f"ip netns exec bcli tc qdisc add dev {dev} root netem {netem}")
else:
sh(f"ip netns exec bcli tc qdisc add dev {dev} root {fq}")
def net_down():
for ns in ("bcli", "bprx", "bsrv"):
@ -959,7 +986,7 @@ def run_http(proxy, params, args, work, mark):
res["rate"] = float(m.group(1)) if m else 0.0
m = re.search(r"Transfer/sec:\s+([\d.]+)(\w+)", out)
res["gbit"] = float(m.group(1)) * UNITS[m.group(2)] * 8 / 1e9 if m else 0.0
for pct in ("50", "99"):
for pct in ("50", "90", "99"):
m = re.search(rf"^\s+{pct}%\s+(\S+)", out, re.M)
res[f"p{pct}_ms"] = to_ms(m.group(1)) if m else float("nan")
errors = 0
@ -1013,7 +1040,7 @@ def run_stream(proxy, params, args, work, mark):
finally:
if wrapper:
stop(wrapper)
res = {"raw": out, "rate": float("nan"), "p50_ms": float("nan"),
res = {"raw": out, "rate": float("nan"), "p50_ms": float("nan"), "p90_ms": float("nan"),
"p99_ms": float("nan"), "requests": 0, "errors": 0, "retrans": 0, "timeouts": 0}
try:
j = json.loads(out)
@ -1087,8 +1114,8 @@ def run_mix(proxy, params, args, work, mark):
res["rate"] = j.get("rate", 0.0)
res["gbit"] = j.get("gbit", 0.0)
res["bytes"] = j.get("bytes", 0)
res["p50_ms"] = j.get("p50_ms", float("nan"))
res["p99_ms"] = j.get("p99_ms", float("nan"))
for pct in ("50", "90", "99"):
res[f"p{pct}_ms"] = j.get(f"p{pct}_ms", float("nan"))
res["timeouts"] = j.get("timeouts", 0)
# connections of the pool which could not be opened count too
res["errors"] = j.get("errors", 0) + max(j.get("pool", 0) - j.get("opened", 0), 0)
@ -1151,7 +1178,7 @@ def measure(proxy, scenario, args, work):
# --- main --------------------------------------------------------------------
FIELDS = ["proxy", "scenario", "rep", "gbit", "rate", "p50_ms", "p99_ms", "errors", "timeouts", "retrans",
FIELDS = ["proxy", "scenario", "rep", "gbit", "rate", "p50_ms", "p90_ms", "p99_ms", "errors", "timeouts", "retrans",
"prx_cpu_s", "prx_task_s", "cpu_s_per_gb", "cpu_us_per_req", "peak_mb",
"cli_util", "prx_util", "srv_util", "cli_thread", "srv_thread", "bound"]
@ -1188,7 +1215,10 @@ def main():
ap.add_argument("--cpus", default="cli=0-3,prx=4,srv=5-7",
help="CPUs per role (default %(default)s)")
ap.add_argument("--mtu", type=int, default=1500)
ap.add_argument("--netem", help='netem on the client link, e.g. "delay 1ms rate 10gbit"')
ap.add_argument("--netem", help='netem on the client link, both ways, e.g. '
'"delay 5ms loss 0.5%%"')
ap.add_argument("--flow-rate", help="the client link's rate for each connection, "
"each way, e.g. 20mbit (fq maxrate)")
ap.add_argument("--out", default="bench/results", help="directory for the CSV and logs")
ap.add_argument("--mix", help="the bench/mix binary for the mix-* scenarios "
"(default: bench/mix/mix-linux-<arch>, see bench/mix/build.sh)")

4
bench/mix/main.go
View File

@ -722,6 +722,7 @@ type summary struct {
Gbit float64 `json:"gbit"`
Bytes uint64 `json:"bytes"`
P50ms float64 `json:"p50_ms"`
P90ms float64 `json:"p90_ms"`
P99ms float64 `json:"p99_ms"`
FirstP50ms float64 `json:"first_p50_ms"`
FirstP99ms float64 `json:"first_p99_ms"`
@ -847,7 +848,8 @@ func runClient(cl *client, active int, rate float64, warmup, duration time.Durat
Bytes: cl.bytes.Load(), Errors: cl.errs.Load(), Timeouts: cl.timeouts.Load(),
Reconnects: cl.reconnects.Load(), Late: late.Load(),
Aborts: cl.aborts.Load(), Resets: cl.resets.Load(), HalfCloses: cl.halfcloses.Load(),
P50ms: cl.complete.quantile(0.5), P99ms: cl.complete.quantile(0.99),
P50ms: cl.complete.quantile(0.5), P90ms: cl.complete.quantile(0.9),
P99ms: cl.complete.quantile(0.99),
FirstP50ms: cl.first.quantile(0.5), FirstP99ms: cl.first.quantile(0.99)}
s.DialError, _ = cl.dialErr.Load().(string)
s.ExchError, _ = cl.exchErr.Load().(string)

19
bench/report.py
View File

@ -27,9 +27,10 @@ of the group whose range (the average of its slowest runs to that of its
fastest) reaches the best one's; * marks the best of the row, both groups
together, in the same way. --split puts the splice group in a table of its own
(* still compares across both), --prefix connect- takes only the scenarios so
named. --latency makes one table of the runs' 50th and 99th percentile
latency for a single CSV instead, ranked by the 99th. --overall marks the best of the row only, in bold and with *: for
tables where one group has a single proxy, its group's best says nothing.
named. --latency makes one table of the runs' 50th, 90th and 99th
percentile latency for a single CSV instead, ranked by the 99th. --overall
marks the best of the row only, in bold and with *: for tables where one
group has a single proxy, its group's best says nothing.
"""
import argparse
@ -237,8 +238,8 @@ def combined(machines, labels, splice, prefix=None, split=False, overall_only=Fa
def latency(path, labels, splice, prefix=None):
"""Latency of one machine's runs: each cell the medians of the runs' 50th
and 99th percentiles, in milliseconds. Bold: the lowest 99th percentile
"""Latency of one machine's runs: each cell the medians of the runs' 50th,
90th (when recorded) and 99th percentiles, in milliseconds. Bold: the lowest 99th percentile
of its group and any other whose runs overlap it; *: the same over the
row. Scenarios without a latency (iperf3 streams) are left out."""
by, proxies, scenarios = {}, [], []
@ -269,7 +270,9 @@ def latency(path, labels, splice, prefix=None):
rule.append(":-:")
head += [labels.get(p, p) for p in g]
rule += ["---:"] * len(g)
out = ["| scenario, p50 / p99 ms | " + " | ".join(head) + " |",
pcts = ["p50_ms", "p90_ms", "p99_ms"] if any(values(rs, "p90_ms") for rs in by.values()) \
else ["p50_ms", "p99_ms"]
out = [f"| scenario, {' / '.join(c[:3] for c in pcts)} ms | " + " | ".join(head) + " |",
"|---|" + "|".join(rule) + "|"]
for s in scenarios:
overall = winners(by, s, everyone, "p99_ms", False)
@ -283,7 +286,7 @@ def latency(path, labels, splice, prefix=None):
if not sel:
cells.append("–")
continue
text = f"{ms(median(sel, 'p50_ms'))} / {ms(median(sel, 'p99_ms'))}"
text = " / ".join(ms(median(sel, c)) for c in pcts)
if p in best:
text = f"**{text}**"
if p in overall:
@ -312,7 +315,7 @@ def main():
ap.add_argument("--splice", action="append", default=[],
help="a proxy (as named in the CSV) using splice(); repeat")
ap.add_argument("--latency", action="store_true",
help="one table of latency, 50th / 99th percentile, for the "
help="one table of latency, 50th / 90th / 99th percentile, for the "
"one CSV given")
args = ap.parse_args()
labels = dict(l.split("=", 1) for l in args.label)