Event loops: a relayed client's buffers given back while empty

A connection's two data buffers are malloc()'d with it (bufalloc())
instead of being part of struct clientparam, which is CLIENTBUFSIZE*2
smaller. The callbacks' relay of a client whose fiber gave up its stack
frees a side's buffer while it is empty and makes it again to read, and
gives both back before sockmap() or the child's tail goes on: a pool of
idle relayed connections holds no buffers. Fibers and threads keep
theirs as before.

mix scenarios, callbacks (-L -L2), peak memory before/after, throughput
the same:
  Intel: relay 972/632 MB, connect 981/581, socks 992/594
  ARM:   relay 1000/673 MB, connect 1000/615, socks 1020/629

struct clientparam's clibufdata and srvbufdata are pointers now:
plugins built against the old structure are not compatible.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/mainclient.c

@@ -161,6 +161,7 @@ static void serveparam(struct clientparam *param){
 static void freeworker(struct clientparam *param){
  releaseparam(param);
  _3proxy_sem_destroy(param->wake);
+ buffree(param);
  free(param);
 }
 
@@ -286,8 +287,16 @@ static int srvspawn(struct srvstart *s, SOCKET client_sock){
  }
  _3proxy_mutex_unlock(&tpool.mutex);
  reused = newparam != NULL;
- if(!reused && (!(newparam = malloc (sizeof(*defparam))) || (!job && _3proxy_sem_init_timed(newparam->wake, 0, 1)))){
+ if(!reused && (newparam = malloc (sizeof(*defparam)))){
-	if(newparam) free(newparam);
+	newparam->clibufdata = newparam->srvbufdata = NULL;
+	newparam->clibuf = newparam->srvbuf = NULL;
+	if(bufalloc(newparam) || (!job && _3proxy_sem_init_timed(newparam->wake, 0, 1))){
+		buffree(newparam);
+		free(newparam);
+		newparam = NULL;
+	}
+ }
+ if(!newparam){
 	if(job) free(job);
 	srv->so._closesocket(srv->so.state, client_sock);
 	defparam->res = 21;
@@ -480,6 +489,7 @@ void releaseparam(struct clientparam * param) {
 
 void freeparam(struct clientparam * param) {
 	releaseparam(param);
+	buffree(param);
 	free(param);
 }
 

src/mainloop.c

@@ -478,6 +478,19 @@ static void cbbuf(struct clientparam *param, int side, unsigned char **buf, unsi
 	}
 }
 
+/* The side's own buffer given back while it is empty, once nothing but the
+   relay can use it: the client's fiber gave up its stack. The next read
+   makes it again. */
+static void cbbufdrop(struct cbmap *m, int side){
+	struct clientparam *param = m->param;
+	unsigned char **own = side? &param->srvbufdata : &param->clibufdata;
+	unsigned char **buf = side? &param->srvbuf : &param->clibuf;
+
+	if(!m->abandoned || !*own || *buf != *own || (side? param->srvinbuf : param->cliinbuf)) return;
+	free(*own);
+	*own = *buf = NULL;
+}
+
 /* what a way has buffered and in its pipe: none once it is not moving */
 static uint64_t cbheld(struct cbmap *m, struct cbway *w){
 	unsigned char *buf;
@@ -620,6 +633,15 @@ static int cbway(struct cbmap *m, struct cbway *w){
 #endif
 	if(w->left > inb && !m->term_r[w->src] && !end && room > 0 &&
 	    ((rs->ready & (POLLIN | POLLHUP | POLLERR)) || cbpending(m, w->src))){
+		/* given back while empty: made again */
+		if(!buf){
+			if(bufget(param, w->src? SERVER : CLIENT)){
+				m->stop = 1;
+				m->stopres = 21;
+				return 1;
+			}
+			cbbuf(param, w->src, &buf, &inbuf, &offset, &size);
+		}
 		n = sf->_recvfrom(param->sostate, from, (char *)buf + *inbuf, (int)CBMIN(room, w->left - inb), 0, NULL, NULL);
 		if(n > 0){
 			*inbuf += n;
@@ -684,6 +706,7 @@ static int cbway(struct cbmap *m, struct cbway *w){
 		m->term_w[w->dst] = 1;
 		sf->_shutdown(param->sostate, to, SHUT_WR);
 	}
+	cbbufdrop(m, w->src);
 	return progress;
 }
 
@@ -859,6 +882,16 @@ static void cbmapfinish(void *arg){
 	int res = m->res, timeo = m->timeosec, usesplice = m->usesplice0;
 
 	free(m);
+	/* the buffers given back while relayed are the child's again: without
+	   them only the end is logged */
+	if(bufalloc(param)){
+		param->res = (res == -2)? 21 : res;
+		param->maptail = NULL;
+		dolog(param, NULL);
+		releaseparam(param);
+		freeparam(param);
+		return;
+	}
 	if(res == -2) res = sockmap(param, timeo, usesplice);
 	param->res = res;
 	if(param->maptail) param->maptail(param, param->maptaildata);
@@ -985,6 +1018,8 @@ static int fibermap(struct clientparam *param, int timeo, int usesplice){
 			m->abandoned = 1;
 			((struct fiberslot *)m->f)->abandoned = 1;
 			m->f = NULL;
+			cbbufdrop(m, 0);
+			cbbufdrop(m, 1);
 		}
 		fiberyield();
 		/* resumed only if not abandoned */

src/mainudp.c

@@ -145,6 +145,13 @@ static struct udpsession *udpnew(struct udpstate *st){
 	return NULL;
  }
  memset(param, 0, sizeof(*param));
+ if(bufalloc(param)){
+	buffree(param);
+	free(param);
+	free(ss);
+	_3proxy_sem_unlock(srv->childslots);
+	return NULL;
+ }
  memcpy(param, st->u.defparam, offsetof(struct clientparam, filters));
  param->clibuf = param->clibufdata;
  param->srvbuf = param->srvbufdata;

src/proxy.h

@@ -359,6 +359,9 @@ int paramstr(unsigned char *field, size_t size, const unsigned char *s);
 /* sets one of a connection's names or credentials; 1 if it is too long */
 #define PARAMSTR(p, f, s) paramstr((p)->f, sizeof((p)->f), (const unsigned char *)(s))
 void bufrelease(struct clientparam * param, DIRECTION which);
+int bufget(struct clientparam * param, DIRECTION which);
+int bufalloc(struct clientparam * param);
+void buffree(struct clientparam * param);
 
 int sockgetlinebuf(struct clientparam * param, DIRECTION which, unsigned char * buf, int bufsize, int delim, int to);
 int getmultiline(struct clientparam * param, DIRECTION which, unsigned char * buf, int bufsize, const char *cap, int *found);

src/proxymain.c

@@ -260,11 +260,19 @@ int WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLine, int
 	if(! (newparam = malloc (sizeof(s.defparam)))){
 		return 2;
 	};
+	newparam->clibufdata = newparam->srvbufdata = NULL;
+	newparam->clibuf = newparam->srvbuf = NULL;
+	if(bufalloc(newparam)){
+		buffree(newparam);
+		free(newparam);
+		return 2;
+	}
 	memcpy(newparam, &s.defparam, offsetof(struct clientparam, filters));
 	newparam->clibuf = newparam->clibufdata;
 	newparam->srvbuf = newparam->srvbufdata;
 	newparam->clibufsize = newparam->srvbufsize = SRVBUFSIZE(&s.srv);
 	childfunc(newparam);
+	buffree(newparam);
 	free(newparam);
 	return 0;
 	

src/sockgetchar.c

@@ -77,6 +77,31 @@ int sockrecvfrom(struct clientparam *param, SOCKET sock, struct sockaddr * sin,
 	}
 }
 
+/* The side's own buffer, made if it is not there, and the one used if
+   none is. 1 if it cannot be had. */
+int bufget(struct clientparam * param, DIRECTION which){
+	unsigned char **own = which? &param->srvbufdata : &param->clibufdata;
+	unsigned char **buf = which? &param->srvbuf : &param->clibuf;
+
+	if(!*own && !(*own = malloc(CLIENTBUFSIZE))) return 1;
+	if(!*buf) *buf = *own;
+	return 0;
+}
+
+/* Both sides' buffers, as bufget(). */
+int bufalloc(struct clientparam * param){
+	return bufget(param, CLIENT) || bufget(param, SERVER);
+}
+
+/* The own buffers freed, with the structure. */
+void buffree(struct clientparam * param){
+	if(param->clibuf == param->clibufdata) param->clibuf = NULL;
+	if(param->srvbuf == param->srvbufdata) param->srvbuf = NULL;
+	free(param->clibufdata);
+	free(param->srvbufdata);
+	param->clibufdata = param->srvbufdata = NULL;
+}
+
 /* Starts an empty buffer over from its beginning, and goes back to the
    connection's own buffer if a filter needed a bigger one. Does nothing
    while anything is left unread. */

src/structures.h

@@ -963,10 +963,12 @@ struct clientparam {
 	   something done at once, never for what has to outlive a call into
 	   anything that could log. */
 	unsigned char	tmpbuf[CLIENTTMPBUFSIZE];
-	/* Each side's data buffer, which clibuf and srvbuf point to unless a
+	/* Each side's data buffer, CLIENTBUFSIZE of malloc's, which clibuf and
-	   filter needed a bigger one. */
+	   srvbuf point to unless a filter needed a bigger one. Made with the
-	unsigned char	clibufdata[CLIENTBUFSIZE],
+	   connection (bufalloc()); a loop's relay gives an empty one back while
-			srvbufdata[CLIENTBUFSIZE];
+	   nothing but the relay can use it, and makes it again to read. */
+	unsigned char	*clibufdata,
+			*srvbufdata;
 };
 
 /* Threads whose connections have ended wait here for the next one, with