mirror of
https://github.com/3proxy/3proxy.git
synced 2026-11-22 23:59:41 +08:00
Benchmarks: measure the proxy under stress, add results
bench.py: - the proxy gets one CPU, the client four, the origin three (8 CPUs), so the proxy is what runs out first - CPU time is the busy time of a role's CPUs from /proc/stat, softirqs included: with IRQ time accounting the cgroup's own figure leaves out much of the network work - every run records which side set its pace (proxy, client, origin), by the busy share of the role's CPUs and of its busiest thread - a 50 KB keep-alive scenario with 2000 clients - haproxy with splice: haproxy::splice-auto and haproxy::splice - wrk timeouts counted apart from errors; tools found by absolute path; leftovers of an interrupted run are cleaned up report.py turns runs into Markdown tables: throughput, the proxy's CPU per gigabyte or per request, its peak memory, splice and copying proxies in groups of their own, the best of each group marked. RESULTS.md has the comparison of 0.9.4, 1.0.1, devel, nginx and haproxy, with and without splice, on an Apple M4 and an Intel Skylake virtual machine, and how it was measured; data/ has the measurements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
94feaf2728
commit
c13c048f3f
11 files changed
+1127
-41
No files matched your search
+178
-28
@@ -27,6 +27,7 @@ import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
CG = "/sys/fs/cgroup/3proxy-bench"
|
||||
@@ -35,7 +36,7 @@ CG = "/sys/fs/cgroup/3proxy-bench"
|
||||
CLI_ADDR, PRX_IN, PRX_OUT, SRV_ADDR = "10.0.1.2", "10.0.1.1", "10.0.2.1", "10.0.2.2"
|
||||
HTTP_PORT, TCP_PORT, ORIGIN_HTTP, ORIGIN_TCP = 3128, 5201, 8080, 5201
|
||||
|
||||
FILES = {"1k": 1024, "100k": 100 * 1024, "1m": 1024 * 1024}
|
||||
FILES = {"1k": 1024, "50k": 50 * 1024, "100k": 100 * 1024, "1m": 1024 * 1024}
|
||||
|
||||
# name: (kind, parameters)
|
||||
SCENARIOS = {
|
||||
@@ -44,10 +45,26 @@ SCENARIOS = {
|
||||
"stream1-down": ("stream", {"streams": 1, "reverse": True}),
|
||||
"http-ka-1k": ("http", {"file": "1k", "conns": 64}),
|
||||
"http-ka-1m": ("http", {"file": "1m", "conns": 16}),
|
||||
"http-ka-50k-2000": ("http", {"file": "50k", "conns": 2000}),
|
||||
"http-close-1k": ("http", {"file": "1k", "conns": 64, "close": True}),
|
||||
}
|
||||
|
||||
|
||||
TOOLS = {}
|
||||
|
||||
|
||||
def tool(name):
|
||||
"""A tool by its absolute path: sudo's secure_path may leave out where
|
||||
it was installed, /usr/local/bin for a wrk built by hand."""
|
||||
if name not in TOOLS:
|
||||
path = shutil.which(name) or shutil.which(
|
||||
name, path="/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin")
|
||||
if not path:
|
||||
sys.exit(f"{name} not found")
|
||||
TOOLS[name] = path
|
||||
return TOOLS[name]
|
||||
|
||||
|
||||
def sh(cmd, check=True, **kw):
|
||||
return subprocess.run(cmd, shell=True, check=check, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw).stdout
|
||||
@@ -142,8 +159,22 @@ def cg_remove(role):
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def cg_kill(role):
|
||||
"""Whatever is left in a role's cgroup, e.g. after an interrupted run."""
|
||||
try:
|
||||
pids = cg_read(f"{CG}/{role}/cgroup.procs").split()
|
||||
except OSError:
|
||||
return
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(int(pid), signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def cg_down():
|
||||
for role in ("cli", "prx", "srv"):
|
||||
cg_kill(role)
|
||||
cg_remove(role)
|
||||
try:
|
||||
os.rmdir(CG)
|
||||
@@ -160,6 +191,60 @@ def cg_usage(role):
|
||||
return 0.0
|
||||
|
||||
|
||||
TICK = os.sysconf("SC_CLK_TCK")
|
||||
|
||||
|
||||
def cpu_busy():
|
||||
"""Busy and total ticks per CPU. Busy counts interrupts and softirqs too:
|
||||
a kernel with IRQ time accounting leaves them out of the cgroup's CPU
|
||||
time, and that is where much of the network work is done."""
|
||||
busy = {}
|
||||
with open("/proc/stat") as f:
|
||||
for line in f:
|
||||
if not line.startswith("cpu") or line.startswith("cpu "):
|
||||
continue
|
||||
name, *v = line.split()
|
||||
v = [int(x) for x in v]
|
||||
# user nice system idle iowait irq softirq steal
|
||||
busy[int(name[3:])] = (v[0] + v[1] + v[2] + v[5] + v[6] + v[7], sum(v[:8]))
|
||||
return busy
|
||||
|
||||
|
||||
def role_busy(before, after, cpus):
|
||||
"""Busy seconds of a role's CPUs, and the share of them that was busy."""
|
||||
b = t = 0
|
||||
for c in cpus:
|
||||
b += after[c][0] - before[c][0]
|
||||
t += after[c][1] - before[c][1]
|
||||
return b / TICK, (b / t if t else 0.0)
|
||||
|
||||
|
||||
def thread_times(role):
|
||||
"""CPU ticks per thread of everything in a role's cgroup."""
|
||||
times = {}
|
||||
try:
|
||||
tids = cg_read(f"{CG}/{role}/cgroup.threads").split()
|
||||
except OSError:
|
||||
return times
|
||||
for tid in tids:
|
||||
try:
|
||||
with open(f"/proc/{tid}/stat") as f:
|
||||
v = f.read().rsplit(")", 1)[1].split()
|
||||
times[tid] = int(v[11]) + int(v[12])
|
||||
except (OSError, IndexError, ValueError):
|
||||
pass
|
||||
return times
|
||||
|
||||
|
||||
def busiest_thread(before, after, wall):
|
||||
"""The busiest thread's share of one CPU: one thread at 100% is a limit
|
||||
however many CPUs its role has (iperf3 runs a stream in a thread)."""
|
||||
top = 0
|
||||
for tid, t in after.items():
|
||||
top = max(top, t - before.get(tid, 0))
|
||||
return top / TICK / wall if wall else 0.0
|
||||
|
||||
|
||||
def cg_peak(role):
|
||||
try:
|
||||
return int(cg_read(f"{CG}/{role}/memory.peak")) / 2**20
|
||||
@@ -183,10 +268,26 @@ def spawn(ns, role, argv, log):
|
||||
stderr=subprocess.STDOUT, preexec_fn=enter)
|
||||
|
||||
|
||||
LAST_CLIENT_THREAD = [0.0]
|
||||
|
||||
|
||||
def run_in(ns, role, argv, timeout):
|
||||
"""Run a load generator to its end. Its busiest thread's share of a CPU
|
||||
is kept in LAST_CLIENT_THREAD, read just before it exits."""
|
||||
p = spawn(ns, role, argv, subprocess.PIPE)
|
||||
out, _ = p.communicate(timeout=timeout)
|
||||
return out.decode("utf-8", "replace")
|
||||
t0 = time.time()
|
||||
top = {}
|
||||
buf = []
|
||||
reader = threading.Thread(target=lambda: buf.append(p.stdout.read()))
|
||||
reader.start()
|
||||
while p.poll() is None:
|
||||
for tid, t in thread_times(role).items():
|
||||
top[tid] = t
|
||||
time.sleep(0.5)
|
||||
reader.join(timeout)
|
||||
wall = time.time() - t0
|
||||
LAST_CLIENT_THREAD[0] = max(top.values()) / TICK / wall if top and wall else 0.0
|
||||
return (buf[0] if buf else b"").decode("utf-8", "replace")
|
||||
|
||||
|
||||
def stop(p):
|
||||
@@ -245,9 +346,11 @@ http {{
|
||||
}}
|
||||
""")
|
||||
log = open(os.path.join(work, "origin.out"), "w")
|
||||
procs = [spawn("bsrv", "srv", ["nginx", "-c", conf, "-p", work], log),
|
||||
spawn("bsrv", "srv", ["iperf3", "-s", "-B", SRV_ADDR, "-p", str(ORIGIN_TCP)], log)]
|
||||
procs = [spawn("bsrv", "srv", [tool("nginx"), "-c", conf, "-p", work], log),
|
||||
spawn("bsrv", "srv", [tool("iperf3"), "-s", "-B", SRV_ADDR, "-p", str(ORIGIN_TCP)], log)]
|
||||
if not (wait_port("bsrv", SRV_ADDR, ORIGIN_HTTP) and wait_port("bsrv", SRV_ADDR, ORIGIN_TCP)):
|
||||
for p in procs:
|
||||
stop(p)
|
||||
sys.exit("the origin did not start, see " + work)
|
||||
return procs
|
||||
|
||||
@@ -271,8 +374,12 @@ tcppm {opts} -i{PRX_IN} -e{PRX_OUT} {TCP_PORT} {SRV_ADDR} {ORIGIN_TCP}
|
||||
conf = os.path.join(work, f"{label}.conf")
|
||||
with open(conf, "w") as f:
|
||||
# distributions build the stream module as a dynamic one
|
||||
stream = "/usr/lib/nginx/modules/ngx_stream_module.so"
|
||||
load = f"load_module {stream};\n" if os.path.exists(stream) else ""
|
||||
load = ""
|
||||
for d in ("/usr/lib/nginx/modules", "/usr/lib64/nginx/modules"):
|
||||
stream = os.path.join(d, "ngx_stream_module.so")
|
||||
if os.path.exists(stream):
|
||||
load = f"load_module {stream};\n"
|
||||
break
|
||||
f.write(f"""{load}daemon off;
|
||||
worker_processes {ncpu};
|
||||
pid {work}/{label}.pid;
|
||||
@@ -303,13 +410,20 @@ stream {{
|
||||
""")
|
||||
return ["-c", conf, "-p", work]
|
||||
if kind == "haproxy":
|
||||
# splice-auto: haproxy's heuristic, splicing a direction once reads
|
||||
# fill its buffer three times in a row; splice: both directions always
|
||||
splice = {"": "", "splice-auto": " option splice-auto\n",
|
||||
"splice": " option splice-request\n option splice-response\n"}
|
||||
if opts not in splice:
|
||||
sys.exit("haproxy options: splice-auto or splice")
|
||||
splice = splice[opts]
|
||||
conf = os.path.join(work, f"{label}.cfg")
|
||||
with open(conf, "w") as f:
|
||||
f.write(f"""global
|
||||
nbthread {ncpu}
|
||||
maxconn 200000
|
||||
defaults
|
||||
timeout connect 5s
|
||||
{splice} timeout connect 5s
|
||||
timeout client 60s
|
||||
timeout server 60s
|
||||
frontend http
|
||||
@@ -344,8 +458,11 @@ class Proxy:
|
||||
self.binary = os.path.abspath(path)
|
||||
self.label = os.path.basename(path) + (self.opts.replace(" ", "") if self.opts else "")
|
||||
elif kind in ("nginx", "haproxy"):
|
||||
self.binary = path or shutil.which(kind)
|
||||
self.label = kind
|
||||
path, _, self.opts = path.partition(":")
|
||||
if kind == "nginx" and self.opts:
|
||||
sys.exit("nginx takes no options (it does not use splice() at all)")
|
||||
self.binary = path or tool(kind)
|
||||
self.label = kind + ("-" + self.opts if self.opts else "")
|
||||
elif kind == "none":
|
||||
self.binary = None
|
||||
self.label = "direct"
|
||||
@@ -400,7 +517,7 @@ def run_http(proxy, params, args, work):
|
||||
SRV_ADDR, ORIGIN_HTTP))
|
||||
threads = len(cpulist(args.cpus["cli"]))
|
||||
conns = max(params["conns"], threads)
|
||||
out = run_in("bcli", "cli", ["wrk", "-t", str(threads), "-c", str(conns),
|
||||
out = run_in("bcli", "cli", [tool("wrk"), "-t", str(threads), "-c", str(conns),
|
||||
"-d", f"{args.duration}s", "--latency", "-s", script,
|
||||
f"http://{host}:{port}/"], args.duration + 30)
|
||||
res = {"raw": out}
|
||||
@@ -414,12 +531,17 @@ def run_http(proxy, params, args, work):
|
||||
m = re.search(rf"^\s+{pct}%\s+(\S+)", out, re.M)
|
||||
res[f"p{pct}_ms"] = to_ms(m.group(1)) if m else float("nan")
|
||||
errors = 0
|
||||
res["timeouts"] = 0
|
||||
m = re.search(r"Socket errors: connect (\d+), read (\d+), write (\d+), timeout (\d+)", out)
|
||||
if m:
|
||||
errors += sum(int(x) for x in m.groups())
|
||||
errors += sum(int(x) for x in m.groups()[:3])
|
||||
# requests slower than wrk's 2 seconds: a latency tail, not a failure
|
||||
res["timeouts"] = int(m.group(4))
|
||||
m = re.search(r"Non-2xx or 3xx responses: (\d+)", out)
|
||||
if m:
|
||||
errors += int(m.group(1))
|
||||
if not res["requests"]:
|
||||
errors += 1 # wrk did not run, or reported nothing
|
||||
res["errors"] = errors
|
||||
res["retrans"] = 0
|
||||
res["bytes"] = res["gbit"] * 1e9 / 8 * args.duration
|
||||
@@ -430,13 +552,13 @@ def run_stream(proxy, params, args, work):
|
||||
direct = proxy.kind == "none"
|
||||
host = SRV_ADDR if direct else PRX_IN
|
||||
port = ORIGIN_TCP if direct else TCP_PORT
|
||||
argv = ["iperf3", "-c", host, "-p", str(port), "-t", str(args.duration),
|
||||
argv = [tool("iperf3"), "-c", host, "-p", str(port), "-t", str(args.duration),
|
||||
"-P", str(params["streams"]), "-J"]
|
||||
if params.get("reverse"):
|
||||
argv.append("-R")
|
||||
out = run_in("bcli", "cli", argv, args.duration + 30)
|
||||
res = {"raw": out, "rate": float("nan"), "p50_ms": float("nan"),
|
||||
"p99_ms": float("nan"), "requests": 0, "errors": 0, "retrans": 0}
|
||||
"p99_ms": float("nan"), "requests": 0, "errors": 0, "retrans": 0, "timeouts": 0}
|
||||
try:
|
||||
j = json.loads(out)
|
||||
s = j["end"]["sum_received"]
|
||||
@@ -450,24 +572,45 @@ def run_stream(proxy, params, args, work):
|
||||
return res
|
||||
|
||||
|
||||
BUSY = 0.9
|
||||
|
||||
|
||||
def measure(proxy, scenario, args, work):
|
||||
kind, params = SCENARIOS[scenario]
|
||||
p = proxy.start(args, work)
|
||||
try:
|
||||
time.sleep(0.5)
|
||||
before = {r: cg_usage(r) for r in ("cli", "prx", "srv")}
|
||||
task0 = cg_usage("prx")
|
||||
cpu0 = cpu_busy()
|
||||
thr0 = {r: thread_times(r) for r in ("cli", "srv")}
|
||||
t0 = time.time()
|
||||
res = (run_http if kind == "http" else run_stream)(proxy, params, args, work)
|
||||
res["cli_thread"] = LAST_CLIENT_THREAD[0]
|
||||
wall = time.time() - t0
|
||||
after = {r: cg_usage(r) for r in ("cli", "prx", "srv")}
|
||||
cpu1 = cpu_busy()
|
||||
# threads of the load generator have ended by now: take what the
|
||||
# client's cgroup reports for the run instead
|
||||
thr1 = {r: thread_times(r) for r in ("cli", "srv")}
|
||||
res["prx_task_s"] = cg_usage("prx") - task0
|
||||
res["peak_mb"] = cg_peak("prx")
|
||||
finally:
|
||||
stop(p)
|
||||
for r in ("cli", "prx", "srv"):
|
||||
used = after[r] - before[r]
|
||||
res[f"{r}_cpu_s"] = used
|
||||
# share of the CPUs given to that role, to see which side was the limit
|
||||
res[f"{r}_util"] = used / (wall * len(cpulist(args.cpus[r])))
|
||||
# the busy time of the role's CPUs, its own processes and the network
|
||||
# work done there for it
|
||||
res[f"{r}_cpu_s"], res[f"{r}_util"] = role_busy(cpu0, cpu1, cpulist(args.cpus[r]))
|
||||
res["srv_thread"] = busiest_thread(thr0["srv"], thr1["srv"], wall)
|
||||
# Which side set the pace: a role whose CPUs, or a single thread of it,
|
||||
# were busy all the time. A run limited by the client or the origin says
|
||||
# only that the proxy can do at least this much.
|
||||
if res["srv_util"] >= BUSY or res["srv_thread"] >= BUSY:
|
||||
res["bound"] = "origin"
|
||||
elif res["cli_util"] >= BUSY or res["cli_thread"] >= BUSY:
|
||||
res["bound"] = "client"
|
||||
elif res["prx_util"] >= BUSY:
|
||||
res["bound"] = "proxy"
|
||||
else:
|
||||
res["bound"] = "none"
|
||||
gb = res["bytes"] / 1e9
|
||||
res["cpu_s_per_gb"] = res["prx_cpu_s"] / gb if gb else float("nan")
|
||||
res["cpu_us_per_req"] = (res["prx_cpu_s"] * 1e6 / res["requests"]
|
||||
@@ -477,9 +620,9 @@ def measure(proxy, scenario, args, work):
|
||||
|
||||
# --- main --------------------------------------------------------------------
|
||||
|
||||
FIELDS = ["proxy", "scenario", "rep", "gbit", "rate", "p50_ms", "p99_ms", "errors", "retrans",
|
||||
"prx_cpu_s", "cpu_s_per_gb", "cpu_us_per_req", "peak_mb",
|
||||
"cli_util", "prx_util", "srv_util"]
|
||||
FIELDS = ["proxy", "scenario", "rep", "gbit", "rate", "p50_ms", "p99_ms", "errors", "timeouts", "retrans",
|
||||
"prx_cpu_s", "prx_task_s", "cpu_s_per_gb", "cpu_us_per_req", "peak_mb",
|
||||
"cli_util", "prx_util", "srv_util", "cli_thread", "srv_thread", "bound"]
|
||||
|
||||
|
||||
def summary(rows):
|
||||
@@ -503,13 +646,14 @@ def summary(rows):
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--proxy", action="append", required=True,
|
||||
help="3proxy:<binary>[:<service options>], nginx[:<binary>], haproxy[:<binary>] "
|
||||
help="3proxy:<binary>[:<service options>], nginx[:<binary>], "
|
||||
"haproxy[:<binary>[:splice-auto|splice]] "
|
||||
"or none (the kernel routing, an upper bound); repeat")
|
||||
ap.add_argument("--scenario", action="append", choices=sorted(SCENARIOS),
|
||||
help="default: all")
|
||||
ap.add_argument("--reps", type=int, default=3)
|
||||
ap.add_argument("--duration", type=int, default=10, help="seconds per run")
|
||||
ap.add_argument("--cpus", default="cli=0-3,prx=4-5,srv=6-7",
|
||||
ap.add_argument("--cpus", default="cli=0-3,prx=4,srv=5-7",
|
||||
help="CPUs per role (default %(default)s)")
|
||||
ap.add_argument("--mtu", type=int, default=1500)
|
||||
ap.add_argument("--netem", help='netem on the client link, e.g. "delay 1ms rate 10gbit"')
|
||||
@@ -520,6 +664,9 @@ def main():
|
||||
args.cpus = dict(x.split("=") for x in args.cpus.split(","))
|
||||
proxies = [Proxy(s) for s in args.proxy]
|
||||
scenarios = args.scenario or list(SCENARIOS)
|
||||
# fail now rather than after the first scenarios
|
||||
for name in ("nginx", "iperf3", "wrk"):
|
||||
tool(name)
|
||||
|
||||
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
@@ -531,7 +678,8 @@ def main():
|
||||
f.write(" ".join(sys.argv) + "\n")
|
||||
f.write(sh("uname -a; lscpu | head -20; nproc; "
|
||||
"cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null; "
|
||||
"iperf3 --version | head -1; wrk -v 2>&1 | head -1; nginx -v 2>&1; "
|
||||
f"{tool('iperf3')} --version | head -1; {tool('wrk')} -v 2>&1 | head -1; "
|
||||
f"{tool('nginx')} -v 2>&1; "
|
||||
"haproxy -v 2>/dev/null | head -1", check=False))
|
||||
|
||||
rows = []
|
||||
@@ -556,12 +704,14 @@ def main():
|
||||
f.flush()
|
||||
raw.write(f"=== {proxy.label} {scenario} {rep}\n{res['raw']}\n")
|
||||
print("%-24s %-14s rep %d: %7.2f Gbit/s %10.0f req/s proxy cpu %5.2fs "
|
||||
"util cli %3.0f%% prx %3.0f%% srv %3.0f%% errors %d" % (
|
||||
"busy cli %3.0f%% prx %3.0f%% srv %3.0f%% (threads %3.0f%%/%3.0f%%) "
|
||||
"bound %-6s errors %d" % (
|
||||
proxy.label, scenario, rep, res["gbit"],
|
||||
res["rate"] if res["rate"] == res["rate"] else 0,
|
||||
res["prx_cpu_s"], res["cli_util"] * 100,
|
||||
res["prx_util"] * 100, res["srv_util"] * 100,
|
||||
res["errors"]), flush=True)
|
||||
res["cli_thread"] * 100, res["srv_thread"] * 100,
|
||||
res["bound"], res["errors"]), flush=True)
|
||||
finally:
|
||||
for p in origin:
|
||||
stop(p)
|
||||
|
||||
Reference in new issue
Block a user