From 5de7ae2dac770b9a7b4905ce3c8bb2b67b7d218b Mon Sep 17 00:00:00 2001 From: Vladimir Dubrovin <3proxy@3proxy.ru> Date: Wed, 28 Oct 2026 17:16:41 +0300 Subject: [PATCH] Buffers: a pool of mapped pages, buffers of 16384 bytes A connection's buffers came from malloc(), which keeps what it is given back: a buffer let go while the connection waits (HTTP keep-alive, an idle relay) left its memory with the process. They now come from a pool of memory mapped for them, in chunks aligned to their size, the first unit of a chunk its header. A unit is a page, or a buffer if pages are smaller: with pages of 4 or 16 KB a buffer is a unit of its own, with pages of 64 KB a unit holds four. Free units are kept for the next buffer, the most recently freed first; more than 64 and an eighth of those in use, and the least recently freed are given back to the system (MADV_DONTNEED on Linux, MADV_FREE_REUSABLE on macOS, MADV_FREE elsewhere, MEM_RESET on Windows), and a chunk given back whole is unmapped. Built for AddressSanitizer the buffers stay malloc()'s, which it checks for overruns, unless WITHBUFPOOL is defined; the pool then poisons the free ones. Buffers are 16384 bytes, not 16384 + 64: a size of pages. The 64 bytes were room for a TLS record's header, which tlspr needs to have a record whole; tlspr now takes a larger buffer when a record does not fit, up to TLS's largest (5 + 16384 + 2048), given back once it is emptied, and relays read the whole buffer again, a record at most as before. A record larger than the buffer set with -b is no longer refused. HTTP proxy, 2000 idle keep-alive connections, after a request (Linux container): 39.6 KB a connection before, 19.5 KB now; a thread a connection, 44.2 KB before, 24.1 KB now; a TCP relay, 13.0 KB before, 11.6 KB now. Incompatible for plugins: CLIENTBUFSIZE is 16384; clibufdata and srvbufdata are the pool's, never to be passed to free() or realloc(). bufpoolinit() is called before any service starts. Co-Authored-By: Claude Opus 5.5 --- src/3proxy.c | 1 + src/mainloop.c | 5 +- src/proxy.h | 1 + src/proxymain.c | 1 + src/sockgetchar.c | 306 +++++++++++++++++++++++++++++++++++++++++++++- src/sockmap.c | 14 +-- src/structures.h | 10 +- src/tlspr.c | 10 +- 8 files changed, 326 insertions(+), 22 deletions(-) diff --git a/src/3proxy.c b/src/3proxy.c index c796225..64d7408 100644 --- a/src/3proxy.c +++ b/src/3proxy.c @@ -426,6 +426,7 @@ int main(int argc, char * argv[]) { _3proxy_mutex_init(&tc_mutex); _3proxy_mutex_init(&loops_mutex); _3proxy_mutex_init(&tpool.mutex); + bufpoolinit(); #ifndef NORADIUS _3proxy_mutex_init(&rad_mutex); #endif diff --git a/src/mainloop.c b/src/mainloop.c index 1f297f6..e5ad7c1 100644 --- a/src/mainloop.c +++ b/src/mainloop.c @@ -499,9 +499,8 @@ struct cbmap { int res; }; -/* as sockmap()'s ROOM() */ -#define CBROOM(size, inbuf) ((((size) == CLIENTBUFSIZE && (inbuf) <= CLIENTBUFSIZE - 64)? \ - (uint64_t)(CLIENTBUFSIZE - 64) : (uint64_t)(size)) - (inbuf)) +/* as sockmap()'s CLIROOM and SRVROOM, what is left of it */ +#define CBROOM(size, inbuf) ((uint64_t)(size) - (inbuf)) #define CBAGAIN(e) ((e) == EAGAIN || (e) == EWOULDBLOCK || (e) == EINTR) #define CBMIN(a, b) ((a) < (b)? (a) : (b)) diff --git a/src/proxy.h b/src/proxy.h index 04e2bee..19da67e 100644 --- a/src/proxy.h +++ b/src/proxy.h @@ -354,6 +354,7 @@ int bufget(struct clientparam * param, DIRECTION which); void bufdrop(struct clientparam * param, DIRECTION which); int bufalloc(struct clientparam * param); void buffree(struct clientparam * param); +void bufpoolinit(void); int sockgetlinebuf(struct clientparam * param, DIRECTION which, unsigned char * buf, int bufsize, int delim, int to); int getmultiline(struct clientparam * param, DIRECTION which, unsigned char * buf, int bufsize, const char *cap, int *found); diff --git a/src/proxymain.c b/src/proxymain.c index edf00d1..8590b13 100644 --- a/src/proxymain.c +++ b/src/proxymain.c @@ -216,6 +216,7 @@ int main (int argc, char** argv){ #endif _3proxy_mutex_init(&tpool.mutex); + bufpoolinit(); loginit(); #ifndef _WIN32 /* a client gone while it is written to is an error, not the end */ diff --git a/src/sockgetchar.c b/src/sockgetchar.c index ffba954..5a78519 100644 --- a/src/sockgetchar.c +++ b/src/sockgetchar.c @@ -77,13 +77,311 @@ int sockrecvfrom(struct clientparam *param, SOCKET sock, struct sockaddr * sin, } } +/* The buffer pool: the connections' own buffers, CLIENTBUFSIZE each, from + memory mapped for them rather than from the heap, which keeps what it is + given back. Mapped in chunks of POOLUNITS units, a chunk aligned to its + size so that a buffer's chunk is its address rounded down; the first + unit holds the chunk's header. A unit is a page, or CLIENTBUFSIZE if + pages are smaller: with pages of 4 or 16 KB a buffer is a unit of its + own, with pages of 64 KB a unit holds four. A unit with all its buffers + free is hot, kept as it is for the next one asked for, the most recently + freed first; more of them than POOLHOT and an eighth of those in use, + and the least recently freed are let go, their pages given back to the + system, and a chunk all of whose units are let go is unmapped. Without + bufpoolinit(), with pages it cannot use or built for AddressSanitizer + (unless WITHBUFPOOL is defined), the buffers are malloc()'s. */ +#if defined __has_feature +#if __has_feature(address_sanitizer) +#define BUFPOOL_ASAN +#endif +#endif +#if defined __SANITIZE_ADDRESS__ && !defined BUFPOOL_ASAN +#define BUFPOOL_ASAN +#endif +#if defined BUFPOOL_ASAN && !defined WITHBUFPOOL && !defined NOBUFPOOL +#define NOBUFPOOL +#endif +#if defined BUFPOOL_ASAN && defined _WIN32 +#undef BUFPOOL_ASAN +#endif +#ifdef BUFPOOL_ASAN +#include +#define POOLPOISON(p, size) ASAN_POISON_MEMORY_REGION((p), (size)) +#define POOLUNPOISON(p, size) ASAN_UNPOISON_MEMORY_REGION((p), (size)) +#else +#define POOLPOISON(p, size) +#define POOLUNPOISON(p, size) +#endif + +#ifndef NOBUFPOOL +#ifndef _WIN32 +#include +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif +#endif + +#define POOLUNITS 64 /* a chunk's units, its header's among them */ +#define POOLHOT 64 /* hot units kept at least */ +#define POOLTRIM 32 /* hot units over the limit before any is let go */ +#define POOLBATCH 64 /* units let go at a time at most */ + +struct poolunit { + struct poolunit *prev, *next; /* in its list, if it is in one */ + unsigned freemask; /* its buffers not in use */ +}; + +struct poolchunk { + unsigned ncold; /* units let go */ + struct poolunit unit[POOLUNITS]; /* unit[0] is the header's */ +}; + +static struct { + _3proxy_mutex_t mutex; + size_t unit, chunk; /* bytes; unit 0 while the pool is not used */ + unsigned all; /* freemask of a unit with all its buffers free */ + unsigned nhot, nused; /* hot units, buffers in use */ + /* list heads: units with buffers both in use and free, hot units + (the most recently freed first), units let go */ + struct poolunit partial, hot, cold; +} pool; + +static void unitlink(struct poolunit *head, struct poolunit *u){ + u->prev = head; + u->next = head->next; + head->next->prev = u; + head->next = u; +} + +static void unitunlink(struct poolunit *u){ + u->prev->next = u->next; + u->next->prev = u->prev; +} + +static struct poolchunk *chunkof(const void *p){ + return (struct poolchunk *)((uintptr_t)p & ~(uintptr_t)(pool.chunk - 1)); +} + +static unsigned char *unitaddr(struct poolunit *u){ + struct poolchunk *c = chunkof(u); + + return (unsigned char *)c + (size_t)(u - c->unit) * pool.unit; +} + +/* pages given back to the system, their addresses kept, and made ready to + be used again */ +static void pagesrelease(void *p, size_t size){ +#ifdef _WIN32 + VirtualAlloc(p, size, MEM_RESET, PAGE_READWRITE); +#elif defined __linux__ || !defined MADV_FREE + /* MADV_FREE leaves them counted to the process until memory is short */ + madvise(p, size, MADV_DONTNEED); +#elif defined MADV_FREE_REUSABLE + madvise(p, size, MADV_FREE_REUSABLE); +#else + madvise(p, size, MADV_FREE); +#endif +} + +static void pagesreuse(void *p, size_t size){ +#if !defined _WIN32 && defined MADV_FREE_REUSE && !defined __linux__ + madvise(p, size, MADV_FREE_REUSE); +#endif +} + +static void chunkunmap(struct poolchunk *c){ + POOLUNPOISON(c, pool.chunk); +#ifdef _WIN32 + VirtualFree(c, 0, MEM_RELEASE); +#else + munmap((void *)c, pool.chunk); +#endif +} + +/* A chunk mapped, its units let go as they are; 1 if it cannot be had. */ +static int chunknew(void){ + struct poolchunk *c = NULL; + unsigned i; +#ifdef _WIN32 + /* reserved twice the size to find an aligned place, which is then + taken unless another thread got there first */ + for(i = 0; i < 16 && !c; i++){ + unsigned char *p = VirtualAlloc(NULL, pool.chunk * 2, MEM_RESERVE, PAGE_NOACCESS); + + if(!p) return 1; + VirtualFree(p, 0, MEM_RELEASE); + c = VirtualAlloc((void *)(((uintptr_t)p + pool.chunk - 1) & ~(uintptr_t)(pool.chunk - 1)), + pool.chunk, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + } + if(!c) return 1; +#else + unsigned char *p, *a; + + p = mmap(NULL, pool.chunk * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if(p == MAP_FAILED) return 1; + a = (unsigned char *)(((uintptr_t)p + pool.chunk - 1) & ~(uintptr_t)(pool.chunk - 1)); + if(a > p) munmap(p, (size_t)(a - p)); + if(a + pool.chunk < p + pool.chunk * 2) munmap(a + pool.chunk, (size_t)(p + pool.chunk * 2 - (a + pool.chunk))); +#ifdef MADV_NOHUGEPAGE + /* a huge page would be taken whole for the first buffer of it */ + madvise(a, pool.chunk, MADV_NOHUGEPAGE); +#endif + c = (struct poolchunk *)a; +#endif + POOLPOISON((unsigned char *)c + pool.unit, pool.chunk - pool.unit); + c->ncold = POOLUNITS - 1; + for(i = 1; i < POOLUNITS; i++){ + c->unit[i].freemask = pool.all; + unitlink(&pool.cold, c->unit + i); + } + return 0; +} + +/* The pool set up, before any buffer is made. */ +void bufpoolinit(void){ + size_t page; + + if(pool.unit) return; +#ifdef _WIN32 + { + SYSTEM_INFO si; + + GetSystemInfo(&si); + page = si.dwPageSize; + } +#else + page = (size_t)sysconf(_SC_PAGESIZE); +#endif + pool.unit = page > CLIENTBUFSIZE? page : CLIENTBUFSIZE; + pool.chunk = pool.unit * POOLUNITS; + /* pages it cannot cut into buffers, or more of them in one than the + mask has room for, or a chunk it cannot align to its size */ + if(!page || pool.unit % page || pool.unit % CLIENTBUFSIZE || pool.unit / CLIENTBUFSIZE > 32 || + (pool.chunk & (pool.chunk - 1)) || sizeof(struct poolchunk) > pool.unit){ + pool.unit = 0; + return; + } + pool.all = (pool.unit / CLIENTBUFSIZE == 32)? ~0U : (1U << (pool.unit / CLIENTBUFSIZE)) - 1; + _3proxy_mutex_init(&pool.mutex); + pool.partial.prev = pool.partial.next = &pool.partial; + pool.hot.prev = pool.hot.next = &pool.hot; + pool.cold.prev = pool.cold.next = &pool.cold; +} + +/* A buffer of the pool's, NULL if it cannot be had. */ +static unsigned char *poolget(void){ + struct poolunit *u; + unsigned char *p; + unsigned i, was; + int cold = 0; + + _3proxy_mutex_lock(&pool.mutex); + if((u = pool.partial.next) == &pool.partial){ + if((u = pool.hot.next) != &pool.hot) pool.nhot--; + else { + if(pool.cold.next == &pool.cold && chunknew()){ + _3proxy_mutex_unlock(&pool.mutex); + return NULL; + } + u = pool.cold.next; + chunkof(u)->ncold--; + cold = 1; + } + } + was = u->freemask; + for(i = 0; !(was & (1U << i)); i++); + u->freemask = was & ~(1U << i); + if(was == pool.all){ + unitunlink(u); + if(u->freemask) unitlink(&pool.partial, u); + } + else if(!u->freemask) unitunlink(u); + pool.nused++; + _3proxy_mutex_unlock(&pool.mutex); + if(cold) pagesreuse(unitaddr(u), pool.unit); + p = unitaddr(u) + (size_t)i * CLIENTBUFSIZE; + POOLUNPOISON(p, CLIENTBUFSIZE); + return p; +} + +/* A buffer given back to the pool, and if too many units are hot, the + least recently freed let go. */ +static void poolput(unsigned char *p){ + struct poolchunk *c = chunkof(p), *gone[POOLBATCH]; + struct poolunit *u, *let[POOLBATCH]; + size_t off = (size_t)(p - (unsigned char *)c); + unsigned i = (unsigned)(off % pool.unit / CLIENTBUFSIZE), was, limit, n = 0, ngone = 0, j, k; + + POOLPOISON(p, CLIENTBUFSIZE); + u = c->unit + off / pool.unit; + _3proxy_mutex_lock(&pool.mutex); + was = u->freemask; + u->freemask = was | (1U << i); + pool.nused--; + if(u->freemask == pool.all){ + if(was) unitunlink(u); + unitlink(&pool.hot, u); + pool.nhot++; + } + else if(!was) unitlink(&pool.partial, u); + limit = POOLHOT + pool.nused / (unsigned)(pool.unit / CLIENTBUFSIZE) / 8; + if(pool.nhot > limit + POOLTRIM){ + /* out of every list while their pages are given back, which is + done unlocked: nothing can take them */ + while(pool.nhot > limit && n < POOLBATCH){ + let[n] = pool.hot.prev; + unitunlink(let[n++]); + pool.nhot--; + } + } + _3proxy_mutex_unlock(&pool.mutex); + if(!n) return; + for(j = 0; j < n; j++) pagesrelease(unitaddr(let[j]), pool.unit); + _3proxy_mutex_lock(&pool.mutex); + for(j = 0; j < n; j++){ + c = chunkof(let[j]); + unitlink(&pool.cold, let[j]); + /* a chunk all let go is unmapped */ + if(++c->ncold == POOLUNITS - 1){ + for(k = 1; k < POOLUNITS; k++) unitunlink(c->unit + k); + gone[ngone++] = c; + } + } + _3proxy_mutex_unlock(&pool.mutex); + for(j = 0; j < ngone; j++) chunkunmap(gone[j]); +} +#else +void bufpoolinit(void){ +} +#endif + +/* A buffer for a side of a connection, CLIENTBUFSIZE, NULL if it cannot be + had. */ +static unsigned char *bufnew(void){ +#ifndef NOBUFPOOL + if(pool.unit) return poolget(); +#endif + return malloc(CLIENTBUFSIZE); +} + +static void bufgone(unsigned char *p){ + if(!p) return; +#ifndef NOBUFPOOL + if(pool.unit){ + poolput(p); + return; + } +#endif + free(p); +} + /* The side's own buffer, made if it is not there, and the one used if none is. 1 if it cannot be had. */ int bufget(struct clientparam * param, DIRECTION which){ unsigned char **own = which? ¶m->srvbufdata : ¶m->clibufdata; unsigned char **buf = which? ¶m->srvbuf : ¶m->clibuf; - if(!*own && !(*own = malloc(CLIENTBUFSIZE))) return 1; + if(!*own && !(*own = bufnew())) return 1; if(!*buf) *buf = *own; return 0; } @@ -97,8 +395,8 @@ int bufalloc(struct clientparam * param){ void buffree(struct clientparam * param){ if(param->clibuf == param->clibufdata) param->clibuf = NULL; if(param->srvbuf == param->srvbufdata) param->srvbuf = NULL; - free(param->clibufdata); - free(param->srvbufdata); + bufgone(param->clibufdata); + bufgone(param->srvbufdata); param->clibufdata = param->srvbufdata = NULL; } @@ -111,7 +409,7 @@ void bufdrop(struct clientparam * param, DIRECTION which){ unsigned *off = which? ¶m->srvoffset : ¶m->clioffset; if(!*own || *buf != *own || *in != *off) return; - free(*own); + bufgone(*own); *own = *buf = NULL; *in = *off = 0; } diff --git a/src/sockmap.c b/src/sockmap.c index 88e3532..6265a5c 100644 --- a/src/sockmap.c +++ b/src/sockmap.c @@ -71,15 +71,11 @@ int pipenew(int p[2], unsigned size){ before the sockets are closed. */ #define SESSIONEND (!halfclose && (CLIENTTERMREAD || SERVERTERMREAD)) -/* The fixed buffers have room for a TLS record and its header, which tlspr - needs. What is relayed is read into them a record at most: written to a - side using TLS, a full buffer then makes one record rather than a record - and another of 64 bytes, which costs a third of the throughput. A buffer - already holding more, or a larger one taken for a filter, is used whole. */ -#define ROOM(size, inbuf) (((size) == CLIENTBUFSIZE && (inbuf) <= CLIENTBUFSIZE - 64)? \ - (unsigned)(CLIENTBUFSIZE - 64) : (unsigned)(size)) -#define CLIROOM ROOM(param->clibufsize, param->cliinbuf) -#define SRVROOM ROOM(param->srvbufsize, param->srvinbuf) +/* What a side's buffer is read into: the whole of it. The fixed buffers + hold a TLS record's data and no more, so written to a side using TLS a + full one makes one record, not a record and another of a few bytes. */ +#define CLIROOM ((unsigned)param->clibufsize) +#define SRVROOM ((unsigned)param->srvbufsize) /* A hang-up on a socket there is data to write to. With poll() it comes once both sides are shut down, and the write tried then fails, which ends diff --git a/src/structures.h b/src/structures.h index 5b9ecbe..e375afe 100644 --- a/src/structures.h +++ b/src/structures.h @@ -155,7 +155,10 @@ typedef struct _3proxy_sem_s { #define MAX_HASH_SIZE (64) #define UDPBUFSIZE 16384 -#define CLIENTBUFSIZE (16384 + 64) +/* A connection's data buffer, either side's: a page or pages of the buffer + pool (bufget()). A TLS record does not always fit: tlspr takes a larger + one for it. */ +#define CLIENTBUFSIZE 16384 #define CLIENTTMPBUFSIZE 8192 /* The defaults of minlogqueue and maxlogqueue: the lines a log's queue keeps slots for, and holds at most; -D at build time sets others. */ @@ -914,8 +917,9 @@ struct clientparam { something done at once, never for what has to outlive a call into anything that could log. */ unsigned char tmpbuf[CLIENTTMPBUFSIZE]; - /* Each side's data buffer, CLIENTBUFSIZE of malloc's, which clibuf and - srvbuf point to unless a filter needed a bigger one. Made with the + /* Each side's data buffer, CLIENTBUFSIZE from the buffer pool, which + clibuf and srvbuf point to unless a filter or a TLS record needed a + bigger one. Made with the connection (bufalloc()); a loop's relay gives an empty one back while nothing but the relay can use it, and makes it again to read. */ unsigned char *clibufdata, diff --git a/src/tlspr.c b/src/tlspr.c index 4612763..f478c97 100644 --- a/src/tlspr.c +++ b/src/tlspr.c @@ -266,9 +266,13 @@ int parsehello(int type, unsigned char *hello, unsigned len, char *sni, int * sn return snifound; } +/* the largest record: TLS 1.2's ciphertext, 2^14 + 2048, and its header */ +#define TLSRECORDMAX (5 + 16384 + 2048) + /* Has one whole TLS record from one side at the start of its buffer, once - whatever is still buffered from the other side is passed on. Returns its - length, or less than 0; the server's codes are one further on than the + whatever is still buffered from the other side is passed on; a buffer + too small for it is made larger, given back once it is emptied. Returns + its length, or less than 0; the server's codes are one further on than the client's, as the log has always had them. */ static int tlstobuf(struct clientparam *param, DIRECTION which){ unsigned *oinbuf = which? ¶m->cliinbuf : ¶m->srvinbuf; @@ -288,7 +292,7 @@ static int tlstobuf(struct clientparam *param, DIRECTION which){ buf = which? param->srvbuf : param->clibuf; if(buf[1] != 3) return -3 - e; len = 5 + size16(buf + 3); - if(len > (which? param->srvbufsize : param->clibufsize)) return -4 - e; + if(len > TLSRECORDMAX || bufroom(param, which, len)) return -4 - e; if(sockfillbuf(param, which, len, param->srv->timeouts[STRING_S]) < len) return -5 - e; return (int)len; }