Benchmarks: measure the proxy under stress, add results

bench.py:
- the proxy gets one CPU, the client four, the origin three (8 CPUs), so
  the proxy is what runs out first
- CPU time is the busy time of a role's CPUs from /proc/stat, softirqs
  included: with IRQ time accounting the cgroup's own figure leaves out
  much of the network work
- every run records which side set its pace (proxy, client, origin), by
  the busy share of the role's CPUs and of its busiest thread
- a 50 KB keep-alive scenario with 2000 clients
- haproxy with splice: haproxy::splice-auto and haproxy::splice
- wrk timeouts counted apart from errors; tools found by absolute path;
  leftovers of an interrupted run are cleaned up

report.py turns runs into Markdown tables: throughput, the proxy's CPU
per gigabyte or per request, its peak memory, splice and copying proxies
in groups of their own, the best of each group marked.

RESULTS.md has the comparison of 0.9.4, 1.0.1, devel, nginx and haproxy,
with and without splice, on an Apple M4 and an Intel Skylake virtual
machine, and how it was measured; data/ has the measurements.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vladimir DubrovinandClaude Opus 5.5 committed 2026-10-11 22:04:47 +03:00
1 parent 94feaf2728
commit c13c048f3f
11 files changed
+1127 -41

No files matched your search

+178 -28
View File
@@ -27,6 +27,7 @@ import statistics
import subprocess
import sys
import tempfile
import threading
import time
CG = "/sys/fs/cgroup/3proxy-bench"
@@ -35,7 +36,7 @@ CG = "/sys/fs/cgroup/3proxy-bench"
CLI_ADDR, PRX_IN, PRX_OUT, SRV_ADDR = "10.0.1.2", "10.0.1.1", "10.0.2.1", "10.0.2.2"
HTTP_PORT, TCP_PORT, ORIGIN_HTTP, ORIGIN_TCP = 3128, 5201, 8080, 5201
FILES = {"1k": 1024, "100k": 100 * 1024, "1m": 1024 * 1024}
FILES = {"1k": 1024, "50k": 50 * 1024, "100k": 100 * 1024, "1m": 1024 * 1024}
# name: (kind, parameters)
SCENARIOS = {
@@ -44,10 +45,26 @@ SCENARIOS = {
"stream1-down": ("stream", {"streams": 1, "reverse": True}),
"http-ka-1k": ("http", {"file": "1k", "conns": 64}),
"http-ka-1m": ("http", {"file": "1m", "conns": 16}),
"http-ka-50k-2000": ("http", {"file": "50k", "conns": 2000}),
"http-close-1k": ("http", {"file": "1k", "conns": 64, "close": True}),
}
TOOLS = {}
def tool(name):
"""A tool by its absolute path: sudo's secure_path may leave out where
it was installed, /usr/local/bin for a wrk built by hand."""
if name not in TOOLS:
path = shutil.which(name) or shutil.which(
name, path="/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin")
if not path:
sys.exit(f"{name} not found")
TOOLS[name] = path
return TOOLS[name]
def sh(cmd, check=True, **kw):
return subprocess.run(cmd, shell=True, check=check, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw).stdout
@@ -142,8 +159,22 @@ def cg_remove(role):
time.sleep(0.1)
def cg_kill(role):
"""Whatever is left in a role's cgroup, e.g. after an interrupted run."""
try:
pids = cg_read(f"{CG}/{role}/cgroup.procs").split()
except OSError:
return
for pid in pids:
try:
os.kill(int(pid), signal.SIGKILL)
except ProcessLookupError:
pass
def cg_down():
for role in ("cli", "prx", "srv"):
cg_kill(role)
cg_remove(role)
try:
os.rmdir(CG)
@@ -160,6 +191,60 @@ def cg_usage(role):
return 0.0
TICK = os.sysconf("SC_CLK_TCK")
def cpu_busy():
"""Busy and total ticks per CPU. Busy counts interrupts and softirqs too:
a kernel with IRQ time accounting leaves them out of the cgroup's CPU
time, and that is where much of the network work is done."""
busy = {}
with open("/proc/stat") as f:
for line in f:
if not line.startswith("cpu") or line.startswith("cpu "):
continue
name, *v = line.split()
v = [int(x) for x in v]
# user nice system idle iowait irq softirq steal
busy[int(name[3:])] = (v[0] + v[1] + v[2] + v[5] + v[6] + v[7], sum(v[:8]))
return busy
def role_busy(before, after, cpus):
"""Busy seconds of a role's CPUs, and the share of them that was busy."""
b = t = 0
for c in cpus:
b += after[c][0] - before[c][0]
t += after[c][1] - before[c][1]
return b / TICK, (b / t if t else 0.0)
def thread_times(role):
"""CPU ticks per thread of everything in a role's cgroup."""
times = {}
try:
tids = cg_read(f"{CG}/{role}/cgroup.threads").split()
except OSError:
return times
for tid in tids:
try:
with open(f"/proc/{tid}/stat") as f:
v = f.read().rsplit(")", 1)[1].split()
times[tid] = int(v[11]) + int(v[12])
except (OSError, IndexError, ValueError):
pass
return times
def busiest_thread(before, after, wall):
"""The busiest thread's share of one CPU: one thread at 100% is a limit
however many CPUs its role has (iperf3 runs a stream in a thread)."""
top = 0
for tid, t in after.items():
top = max(top, t - before.get(tid, 0))
return top / TICK / wall if wall else 0.0
def cg_peak(role):
try:
return int(cg_read(f"{CG}/{role}/memory.peak")) / 2**20
@@ -183,10 +268,26 @@ def spawn(ns, role, argv, log):
stderr=subprocess.STDOUT, preexec_fn=enter)
LAST_CLIENT_THREAD = [0.0]
def run_in(ns, role, argv, timeout):
"""Run a load generator to its end. Its busiest thread's share of a CPU
is kept in LAST_CLIENT_THREAD, read just before it exits."""
p = spawn(ns, role, argv, subprocess.PIPE)
out, _ = p.communicate(timeout=timeout)
return out.decode("utf-8", "replace")
t0 = time.time()
top = {}
buf = []
reader = threading.Thread(target=lambda: buf.append(p.stdout.read()))
reader.start()
while p.poll() is None:
for tid, t in thread_times(role).items():
top[tid] = t
time.sleep(0.5)
reader.join(timeout)
wall = time.time() - t0
LAST_CLIENT_THREAD[0] = max(top.values()) / TICK / wall if top and wall else 0.0
return (buf[0] if buf else b"").decode("utf-8", "replace")
def stop(p):
@@ -245,9 +346,11 @@ http {{
}}
""")
log = open(os.path.join(work, "origin.out"), "w")
procs = [spawn("bsrv", "srv", ["nginx", "-c", conf, "-p", work], log),
spawn("bsrv", "srv", ["iperf3", "-s", "-B", SRV_ADDR, "-p", str(ORIGIN_TCP)], log)]
procs = [spawn("bsrv", "srv", [tool("nginx"), "-c", conf, "-p", work], log),
spawn("bsrv", "srv", [tool("iperf3"), "-s", "-B", SRV_ADDR, "-p", str(ORIGIN_TCP)], log)]
if not (wait_port("bsrv", SRV_ADDR, ORIGIN_HTTP) and wait_port("bsrv", SRV_ADDR, ORIGIN_TCP)):
for p in procs:
stop(p)
sys.exit("the origin did not start, see " + work)
return procs
@@ -271,8 +374,12 @@ tcppm {opts} -i{PRX_IN} -e{PRX_OUT} {TCP_PORT} {SRV_ADDR} {ORIGIN_TCP}
conf = os.path.join(work, f"{label}.conf")
with open(conf, "w") as f:
# distributions build the stream module as a dynamic one
stream = "/usr/lib/nginx/modules/ngx_stream_module.so"
load = f"load_module {stream};\n" if os.path.exists(stream) else ""
load = ""
for d in ("/usr/lib/nginx/modules", "/usr/lib64/nginx/modules"):
stream = os.path.join(d, "ngx_stream_module.so")
if os.path.exists(stream):
load = f"load_module {stream};\n"
break
f.write(f"""{load}daemon off;
worker_processes {ncpu};
pid {work}/{label}.pid;
@@ -303,13 +410,20 @@ stream {{
""")
return ["-c", conf, "-p", work]
if kind == "haproxy":
# splice-auto: haproxy's heuristic, splicing a direction once reads
# fill its buffer three times in a row; splice: both directions always
splice = {"": "", "splice-auto": " option splice-auto\n",
"splice": " option splice-request\n option splice-response\n"}
if opts not in splice:
sys.exit("haproxy options: splice-auto or splice")
splice = splice[opts]
conf = os.path.join(work, f"{label}.cfg")
with open(conf, "w") as f:
f.write(f"""global
nbthread {ncpu}
maxconn 200000
defaults
timeout connect 5s
{splice} timeout connect 5s
timeout client 60s
timeout server 60s
frontend http
@@ -344,8 +458,11 @@ class Proxy:
self.binary = os.path.abspath(path)
self.label = os.path.basename(path) + (self.opts.replace(" ", "") if self.opts else "")
elif kind in ("nginx", "haproxy"):
self.binary = path or shutil.which(kind)
self.label = kind
path, _, self.opts = path.partition(":")
if kind == "nginx" and self.opts:
sys.exit("nginx takes no options (it does not use splice() at all)")
self.binary = path or tool(kind)
self.label = kind + ("-" + self.opts if self.opts else "")
elif kind == "none":
self.binary = None
self.label = "direct"
@@ -400,7 +517,7 @@ def run_http(proxy, params, args, work):
SRV_ADDR, ORIGIN_HTTP))
threads = len(cpulist(args.cpus["cli"]))
conns = max(params["conns"], threads)
out = run_in("bcli", "cli", ["wrk", "-t", str(threads), "-c", str(conns),
out = run_in("bcli", "cli", [tool("wrk"), "-t", str(threads), "-c", str(conns),
"-d", f"{args.duration}s", "--latency", "-s", script,
f"http://{host}:{port}/"], args.duration + 30)
res = {"raw": out}
@@ -414,12 +531,17 @@ def run_http(proxy, params, args, work):
m = re.search(rf"^\s+{pct}%\s+(\S+)", out, re.M)
res[f"p{pct}_ms"] = to_ms(m.group(1)) if m else float("nan")
errors = 0
res["timeouts"] = 0
m = re.search(r"Socket errors: connect (\d+), read (\d+), write (\d+), timeout (\d+)", out)
if m:
errors += sum(int(x) for x in m.groups())
errors += sum(int(x) for x in m.groups()[:3])
# requests slower than wrk's 2 seconds: a latency tail, not a failure
res["timeouts"] = int(m.group(4))
m = re.search(r"Non-2xx or 3xx responses: (\d+)", out)
if m:
errors += int(m.group(1))
if not res["requests"]:
errors += 1 # wrk did not run, or reported nothing
res["errors"] = errors
res["retrans"] = 0
res["bytes"] = res["gbit"] * 1e9 / 8 * args.duration
@@ -430,13 +552,13 @@ def run_stream(proxy, params, args, work):
direct = proxy.kind == "none"
host = SRV_ADDR if direct else PRX_IN
port = ORIGIN_TCP if direct else TCP_PORT
argv = ["iperf3", "-c", host, "-p", str(port), "-t", str(args.duration),
argv = [tool("iperf3"), "-c", host, "-p", str(port), "-t", str(args.duration),
"-P", str(params["streams"]), "-J"]
if params.get("reverse"):
argv.append("-R")
out = run_in("bcli", "cli", argv, args.duration + 30)
res = {"raw": out, "rate": float("nan"), "p50_ms": float("nan"),
"p99_ms": float("nan"), "requests": 0, "errors": 0, "retrans": 0}
"p99_ms": float("nan"), "requests": 0, "errors": 0, "retrans": 0, "timeouts": 0}
try:
j = json.loads(out)
s = j["end"]["sum_received"]
@@ -450,24 +572,45 @@ def run_stream(proxy, params, args, work):
return res
BUSY = 0.9
def measure(proxy, scenario, args, work):
kind, params = SCENARIOS[scenario]
p = proxy.start(args, work)
try:
time.sleep(0.5)
before = {r: cg_usage(r) for r in ("cli", "prx", "srv")}
task0 = cg_usage("prx")
cpu0 = cpu_busy()
thr0 = {r: thread_times(r) for r in ("cli", "srv")}
t0 = time.time()
res = (run_http if kind == "http" else run_stream)(proxy, params, args, work)
res["cli_thread"] = LAST_CLIENT_THREAD[0]
wall = time.time() - t0
after = {r: cg_usage(r) for r in ("cli", "prx", "srv")}
cpu1 = cpu_busy()
# threads of the load generator have ended by now: take what the
# client's cgroup reports for the run instead
thr1 = {r: thread_times(r) for r in ("cli", "srv")}
res["prx_task_s"] = cg_usage("prx") - task0
res["peak_mb"] = cg_peak("prx")
finally:
stop(p)
for r in ("cli", "prx", "srv"):
used = after[r] - before[r]
res[f"{r}_cpu_s"] = used
# share of the CPUs given to that role, to see which side was the limit
res[f"{r}_util"] = used / (wall * len(cpulist(args.cpus[r])))
# the busy time of the role's CPUs, its own processes and the network
# work done there for it
res[f"{r}_cpu_s"], res[f"{r}_util"] = role_busy(cpu0, cpu1, cpulist(args.cpus[r]))
res["srv_thread"] = busiest_thread(thr0["srv"], thr1["srv"], wall)
# Which side set the pace: a role whose CPUs, or a single thread of it,
# were busy all the time. A run limited by the client or the origin says
# only that the proxy can do at least this much.
if res["srv_util"] >= BUSY or res["srv_thread"] >= BUSY:
res["bound"] = "origin"
elif res["cli_util"] >= BUSY or res["cli_thread"] >= BUSY:
res["bound"] = "client"
elif res["prx_util"] >= BUSY:
res["bound"] = "proxy"
else:
res["bound"] = "none"
gb = res["bytes"] / 1e9
res["cpu_s_per_gb"] = res["prx_cpu_s"] / gb if gb else float("nan")
res["cpu_us_per_req"] = (res["prx_cpu_s"] * 1e6 / res["requests"]
@@ -477,9 +620,9 @@ def measure(proxy, scenario, args, work):
# --- main --------------------------------------------------------------------
FIELDS = ["proxy", "scenario", "rep", "gbit", "rate", "p50_ms", "p99_ms", "errors", "retrans",
"prx_cpu_s", "cpu_s_per_gb", "cpu_us_per_req", "peak_mb",
"cli_util", "prx_util", "srv_util"]
FIELDS = ["proxy", "scenario", "rep", "gbit", "rate", "p50_ms", "p99_ms", "errors", "timeouts", "retrans",
"prx_cpu_s", "prx_task_s", "cpu_s_per_gb", "cpu_us_per_req", "peak_mb",
"cli_util", "prx_util", "srv_util", "cli_thread", "srv_thread", "bound"]
def summary(rows):
@@ -503,13 +646,14 @@ def summary(rows):
def main():
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("--proxy", action="append", required=True,
help="3proxy:<binary>[:<service options>], nginx[:<binary>], haproxy[:<binary>] "
help="3proxy:<binary>[:<service options>], nginx[:<binary>], "
"haproxy[:<binary>[:splice-auto|splice]] "
"or none (the kernel routing, an upper bound); repeat")
ap.add_argument("--scenario", action="append", choices=sorted(SCENARIOS),
help="default: all")
ap.add_argument("--reps", type=int, default=3)
ap.add_argument("--duration", type=int, default=10, help="seconds per run")
ap.add_argument("--cpus", default="cli=0-3,prx=4-5,srv=6-7",
ap.add_argument("--cpus", default="cli=0-3,prx=4,srv=5-7",
help="CPUs per role (default %(default)s)")
ap.add_argument("--mtu", type=int, default=1500)
ap.add_argument("--netem", help='netem on the client link, e.g. "delay 1ms rate 10gbit"')
@@ -520,6 +664,9 @@ def main():
args.cpus = dict(x.split("=") for x in args.cpus.split(","))
proxies = [Proxy(s) for s in args.proxy]
scenarios = args.scenario or list(SCENARIOS)
# fail now rather than after the first scenarios
for name in ("nginx", "iperf3", "wrk"):
tool(name)
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
os.makedirs(args.out, exist_ok=True)
@@ -531,7 +678,8 @@ def main():
f.write(" ".join(sys.argv) + "\n")
f.write(sh("uname -a; lscpu | head -20; nproc; "
"cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null; "
"iperf3 --version | head -1; wrk -v 2>&1 | head -1; nginx -v 2>&1; "
f"{tool('iperf3')} --version | head -1; {tool('wrk')} -v 2>&1 | head -1; "
f"{tool('nginx')} -v 2>&1; "
"haproxy -v 2>/dev/null | head -1", check=False))
rows = []
@@ -556,12 +704,14 @@ def main():
f.flush()
raw.write(f"=== {proxy.label} {scenario} {rep}\n{res['raw']}\n")
print("%-24s %-14s rep %d: %7.2f Gbit/s %10.0f req/s proxy cpu %5.2fs "
"util cli %3.0f%% prx %3.0f%% srv %3.0f%% errors %d" % (
"busy cli %3.0f%% prx %3.0f%% srv %3.0f%% (threads %3.0f%%/%3.0f%%) "
"bound %-6s errors %d" % (
proxy.label, scenario, rep, res["gbit"],
res["rate"] if res["rate"] == res["rate"] else 0,
res["prx_cpu_s"], res["cli_util"] * 100,
res["prx_util"] * 100, res["srv_util"] * 100,
res["errors"]), flush=True)
res["cli_thread"] * 100, res["srv_thread"] * 100,
res["bound"], res["errors"]), flush=True)
finally:
for p in origin:
stop(p)