Event loops: a fiber's stack given back while its client is relayed

A child for which mapping is the last thing it does (tcppm, tlspr) sets
maptail, what it does once mapping is over as its own end would: its
log line, prepared before for tlspr. Relayed by the loop, the client's
fiber then gives up its stack at once: left in the middle of the
child's code, never to be resumed, it is started over from the start of
its stack for the next client (fiberreset), or freed where it cannot be
(Windows). Once mapping is over the loop ends the client as the child
and its fiber would: the result, the tail, the connection released and
freed. After a reload, which sockmap() is to see to, the client is
given a fiber again for sockmap() to go on with the mapping, then its
end. Not while a reload is pending: sockmap() sees to it on the fiber.

The callbacks' state is no longer on the fiber's stack but allocated.

clientparam has maptail and maptaildata: plugins are to be built again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/fiber.c

@@ -30,6 +30,7 @@
 #endif
 #ifdef FIBER_ASAN
 #include <sanitizer/common_interface_defs.h>
+#include <sanitizer/asan_interface.h>
 #endif
 
 #ifdef _MSC_VER
@@ -348,8 +349,30 @@ void fiberfree(struct fiber *f){
 	memset(f, 0, sizeof(*f));
 }
 
+/* f, left in the middle of func, to be started again from the start of
+   its stack: whatever its stack held is forgotten. 0, or 1 if it cannot
+   be (Windows), and is to be freed instead. */
+int fiberreset(struct fiber *f){
+#ifdef FIBER_WINDOWS
+	return 1;
+#else
+	f->func = NULL;
+	f->started = 0;
+	f->fake = NULL;
+#ifdef FIBER_ASAN
+	/* what AddressSanitizer knew of the frames left there */
+	{
+		size_t page = (size_t)sysconf(_SC_PAGESIZE);
+
+		__asan_unpoison_memory_region(f->map + page, f->mapped - page);
+	}
+#endif
+	return 0;
+#endif
+}
+
 /* func(arg) is what f runs once resumed: f is new, or func of its last
-   start returned. */
+   start returned, or it was reset. */
 void fiberstart(struct fiber *f, void (*func)(void *arg), void *arg){
 	f->func = func;
 	f->arg = arg;

src/fiber.h

@@ -52,6 +52,7 @@ int fiberusable(void);
 int fiberinit(struct fiber *f, size_t stacksize);
 void fiberfree(struct fiber *f);
 void fiberstart(struct fiber *f, void (*func)(void *arg), void *arg);
+int fiberreset(struct fiber *f);
 int fiberresume(struct fiber *f);
 void fiberyield(void);
 struct fiber *fibercurrent(void);

src/mainclient.c

@@ -385,6 +385,11 @@ static int srvspawn(struct srvstart *s, SOCKET client_sock){
    what was freed. After this the service may be gone at any moment, so
    param->srv is cleared and nothing here touches it again. */
 void releaseparam(struct clientparam * param) {
+	param->maptail = NULL;
+	if(param->maptaildata){
+		free(param->maptaildata);
+		param->maptaildata = NULL;
+	}
 	if(param->srv){
 		if(param->srv->so.freefunc) param->srv->so.freefunc(param->sostate);
 	}

src/mainloop.c

@@ -38,6 +38,9 @@ struct fiberslot {
 	struct fiber f;
 	size_t stacksize;
 	struct fiberslot *next;
+	/* left in the middle of what it runs, never to be resumed: its
+	   client's mapping is the loop's (maptail) */
+	int abandoned;
 };
 
 /* With epoll and kqueue a socket a fiber waits on is registered once, for
@@ -108,6 +111,7 @@ static struct fiberslot *fibertake(struct looper *lp, size_t stacksize){
 		}
 	}
 	if(!(fs = malloc(sizeof(*fs)))) return NULL;
+	fs->abandoned = 0;
 	if(fiberinit(&fs->f, stacksize)){
 		free(fs);
 		return NULL;
@@ -130,7 +134,19 @@ static void fibergive(struct looper *lp, struct fiberslot *fs){
 /* Runs a fiber of this loop until it waits, or gives it back once its
    client is done. */
 static void fiberrun(struct looper *lp, struct fiber *f){
-	if(!fiberresume(f)) fibergive(lp, (struct fiberslot *)f);
+	struct fiberslot *fs = (struct fiberslot *)f;
+
+	if(!fiberresume(f)) fibergive(lp, fs);
+	/* in the middle of its client's code: started over from its stack's
+	   start for the next client, or freed where it cannot be */
+	else if(fs->abandoned){
+		fs->abandoned = 0;
+		if(!fiberreset(&fs->f)) fibergive(lp, fs);
+		else {
+			fiberfree(&fs->f);
+			free(fs);
+		}
+	}
 }
 
 /* A client, on a fiber: as its thread would serve it. */
@@ -424,6 +440,11 @@ struct cbmap {
 	int sleep;		/* ms to pause for: bandwidth, grace */
 	int paused;
 	int stop, stopres;	/* a filter's verdict ends the mapping */
+	/* the client's fiber gave up its stack: the child's end (maptail) is
+	   the loop's once mapping is over */
+	int abandoned;
+	int timeosec, usesplice0;	/* sockmap()'s, for a fiber to go on with
+					   after a reload */
 	int res;
 };
 
@@ -798,13 +819,15 @@ static int cbmaprun(struct cbmap *m){
 	return 0;
 }
 
+static void cbmapdone(struct cbmap *m);
+
 static void cbmapio(void *data){
 	struct cbmap *m = (struct cbmap *)data;
 
 	/* paused: what is ready waits for the pause to end */
 	if(m->paused) return;
-	/* m is on the fiber's stack: not touched once it runs */
-	if(cbmaprun(m)) fiberrun(m->lp, m->f);
+	/* m is not touched once the mapping is over: it may be gone */
+	if(cbmaprun(m)) cbmapdone(m);
 }
 
 static void cbmapagain(struct loop *l, struct looptimer *t){
@@ -822,7 +845,54 @@ static void cbmapidle(struct loop *l, struct looptimer *t){
 		return;
 	}
 	cbmapend(m, 92);
-	fiberrun(m->lp, m->f);
+	cbmapdone(m);
+}
+
+/* What the child of the client does once mapping is over, as its end and
+   that of its fiber would: the result, its tail, the connection released
+   and freed. */
+static void cbmapfinish(struct clientparam *param, int res){
+	param->res = res;
+	if(param->maptail) param->maptail(param, param->maptaildata);
+	releaseparam(param);
+	freeparam(param);
+}
+
+/* a fiber for the client again: sockmap() goes on with the mapping after
+   a reload, then the child's end */
+static void cbmapresume(void *arg){
+	struct cbmap *m = (struct cbmap *)arg;
+	struct clientparam *param = m->param;
+	int timeo = m->timeosec, usesplice = m->usesplice0;
+
+	free(m);
+	cbmapfinish(param, sockmap(param, timeo, usesplice));
+}
+
+/* The mapping is over: its fiber goes on, or, with none, the loop ends it
+   as the child would, or gives it a fiber again for sockmap() to take it
+   over after a reload. m is freed. */
+static void cbmapdone(struct cbmap *m){
+	struct clientparam *param = m->param;
+	struct fiberslot *fs;
+	int res;
+
+	if(!m->abandoned){
+		/* the fiber frees m */
+		fiberrun(m->lp, m->f);
+		return;
+	}
+	if(m->res == -2){
+		if((fs = fibertake(m->lp, param->stacksize))){
+			fiberstart(&fs->f, cbmapresume, m);
+			fiberrun(m->lp, &fs->f);
+			return;
+		}
+		m->res = 21;
+	}
+	res = m->res;
+	free(m);
+	cbmapfinish(param, res);
 }
 
 /* sockmapcbfunc: sockmap() of a fiber, as callbacks, its result; -2 for
@@ -831,8 +901,8 @@ static void cbmapidle(struct loop *l, struct looptimer *t){
    its filters forbid. */
 static int fibermap(struct clientparam *param, int timeo, int usesplice){
 	struct looper *lp = curlooper;
-	struct cbmap m;
-	int i;
+	struct cbmap *m;
+	int i, res, usesplice0 = usesplice;
 
 	if(!lp || !fibercurrent() || param->srv->loopmode != 2) return -2;
 	if(param->clisock == INVALID_SOCKET || param->remsock == INVALID_SOCKET ||
@@ -850,30 +920,33 @@ static int fibermap(struct clientparam *param, int timeo, int usesplice){
 	   socket, data is copied */
 	if(param->srv->so._recvfrom != so._recvfrom || param->srv->so._sendto != so._sendto)
 		usesplice = 0;
-	memset(&m, 0, sizeof(m));
-	m.sock[0] = param->clisock;
-	m.sock[1] = param->remsock;
-	if(!(m.reg[0] = sockreg(lp, m.sock[0])) || !(m.reg[1] = sockreg(lp, m.sock[1]))) return -2;
-	if((param->ctrlsock != INVALID_SOCKET && !(m.ctrl[0] = sockreg(lp, param->ctrlsock))) ||
-	    (param->ctrlsocksrv != INVALID_SOCKET && !(m.ctrl[1] = sockreg(lp, param->ctrlsocksrv))))
+	/* not on the fiber's stack: the fiber may give it up */
+	if(!(m = calloc(1, sizeof(*m)))) return -2;
+	m->sock[0] = param->clisock;
+	m->sock[1] = param->remsock;
+	if(!(m->reg[0] = sockreg(lp, m->sock[0])) || !(m->reg[1] = sockreg(lp, m->sock[1])) ||
+	    (param->ctrlsock != INVALID_SOCKET && !(m->ctrl[0] = sockreg(lp, param->ctrlsock))) ||
+	    (param->ctrlsocksrv != INVALID_SOCKET && !(m->ctrl[1] = sockreg(lp, param->ctrlsocksrv)))){
+		free(m);
 		return -2;
-	m.param = param;
-	m.lp = lp;
-	m.f = fibercurrent();
-	m.halfclose = param->srv->halfclose;
-	m.usesplice = usesplice;
-	m.timeo = (uint64_t)timeo * 1000;
-	m.linger = (uint64_t)param->srv->timeouts[LINGER_TO] * 1000;
-	m.way[0].src = m.way[1].dst = 0;
-	m.way[0].dst = m.way[1].src = 1;
-	m.way[0].left = m.way[1].left = CBALL;
+	}
+	m->param = param;
+	m->lp = lp;
+	m->f = fibercurrent();
+	m->halfclose = param->srv->halfclose;
+	m->usesplice = usesplice;
+	m->timeo = (uint64_t)timeo * 1000;
+	m->linger = (uint64_t)param->srv->timeouts[LINGER_TO] * 1000;
+	m->way[0].src = m->way[1].dst = 0;
+	m->way[0].dst = m->way[1].src = 1;
+	m->way[0].left = m->way[1].left = CBALL;
 	if(param->waitclient64){
-		m.way[0].left = param->waitclient64;
-		m.way[1].left = 0;
+		m->way[0].left = param->waitclient64;
+		m->way[1].left = 0;
 	}
 	if(param->waitserver64){
-		m.way[1].left = param->waitserver64;
-		m.way[0].left = 0;
+		m->way[1].left = param->waitserver64;
+		m->way[0].left = 0;
 	}
 	if(param->srvinbuf == param->srvoffset) param->srvinbuf = param->srvoffset = 0;
 	if(param->cliinbuf == param->clioffset) param->cliinbuf = param->clioffset = 0;
@@ -884,27 +957,41 @@ static int fibermap(struct clientparam *param, int timeo, int usesplice){
 		if(!(usesplice & bit)) continue;
 		/* told to, or a body of known length large enough: spliced from
 		   the start */
-		if(!(usesplice & SPLICE_AUTO) || (m.way[i].left != CBALL &&
-		    m.way[i].left >= (uint64_t)param->srv->splicemin))
-			m.way[i].splice = 1;
+		if(!(usesplice & SPLICE_AUTO) || (m->way[i].left != CBALL &&
+		    m->way[i].left >= (uint64_t)param->srv->splicemin))
+			m->way[i].splice = 1;
 	}
 #endif
-	m.last = looptime();
-	m.idle.func = cbmapidle;
-	m.idle.data = &m;
-	m.again.func = cbmapagain;
-	m.again.data = &m;
-	m.reg[0]->cb = m.reg[1]->cb = cbmapio;
-	m.reg[0]->cbdata = m.reg[1]->cbdata = &m;
-	for(i = 0; i < 2; i++) if(m.ctrl[i]){
-		m.ctrl[i]->cb = cbmapio;
-		m.ctrl[i]->cbdata = &m;
+	m->timeosec = timeo;
+	m->usesplice0 = usesplice0;
+	m->last = looptime();
+	m->idle.func = cbmapidle;
+	m->idle.data = m;
+	m->again.func = cbmapagain;
+	m->again.data = m;
+	m->reg[0]->cb = m->reg[1]->cb = cbmapio;
+	m->reg[0]->cbdata = m->reg[1]->cbdata = m;
+	for(i = 0; i < 2; i++) if(m->ctrl[i]){
+		m->ctrl[i]->cb = cbmapio;
+		m->ctrl[i]->cbdata = m;
 	}
-	looptimerset(&lp->l, &m.idle, cbmapdeadline(&m));
+	looptimerset(&lp->l, &m->idle, cbmapdeadline(m));
 	/* what is ready already is moved at once, on the fiber's stack */
-	if(cbmaprun(&m)) return m.res;
-	fiberyield();
-	return m.res;
+	if(!cbmaprun(m)){
+		/* mapping the child's last thing, its end is the loop's then,
+		   and its stack given back; not while a reload is pending,
+		   which sockmap() is to see to on this fiber */
+		if(param->maptail && param->version >= runconf->version){
+			m->abandoned = 1;
+			((struct fiberslot *)m->f)->abandoned = 1;
+			m->f = NULL;
+		}
+		fiberyield();
+		/* resumed only if not abandoned */
+	}
+	res = m->res;
+	free(m);
+	return res;
 }
 #endif
 

src/structures.h

@@ -937,6 +937,12 @@ struct clientparam {
 			password[USERNAMELEN],
 			extusername[USERNAMELEN],
 			extpassword[USERNAMELEN];
+	/* Set by a child for which mapping is the last thing it does: what it
+	   does once mapping returns, as its own end would, given its result in
+	   res, for a loop to end it so with no fiber of its own meanwhile.
+	   maptaildata is malloc()'d or NULL, freed with the connection. */
+	void (*maptail)(struct clientparam *param, void *data);
+	void *maptaildata;
 	/* The filters this connection runs, which a new connection makes for
 	   itself: the service's are not copied to it. Ordered for each stage. */
 	struct filterp	filters[MAX_FILTERS],

src/tcppm.c

@@ -13,6 +13,11 @@
 #endif
 #define RETURN(xxx) { param->res = xxx; goto CLEANRET; }
 
+/* as the end below, once mapping is over */
+static void tcppmtail(struct clientparam *param, void *data){
+ dolog(param, *param->hostname? param->hostname : NULL);
+}
+
 void * tcppmchild(struct clientparam* param) {
  int res;
 
@@ -43,6 +48,7 @@ void * tcppmchild(struct clientparam* param) {
     return (void *)param->redirectfunc;
  }
 
+ param->maptail = tcppmtail;
  RETURN (mapsocket(param, param->srv->timeouts[CONNECTION_L]));
 CLEANRET:
  

src/tlspr.c

@@ -293,6 +293,11 @@ static int tlstobuf(struct clientparam *param, DIRECTION which){
     return (int)len;
 }
 
+/* as the end below, once mapping is over: its log line, made before */
+static void tlsprtail(struct clientparam *param, void *data){
+ dolog(param, (unsigned char *)data);
+}
+
 void * tlsprchild(struct clientparam* param) {
  int res;
  char sni[SNILEN];
@@ -417,6 +422,8 @@ void * tlsprchild(struct clientparam* param) {
     }
  }
 
+ sprintf(req, "%sv%d.%d %s %s", lv<0?"NONE":lv?"TLS":"SSL", lv<0?0:lv?1:3, lv<0?0:lv?lv-1:0, *param->hostname?(char *)param->hostname:"-", proto);
+ if((param->maptaildata = strdup(req))) param->maptail = tlsprtail;
  RETURN (mapsocket(param, param->srv->timeouts[CONNECTION_L]));
 CLEANRET: