Event loops: readiness answered once is asked again, bit by bit

What a socket was reported ready for is trusted once: answered again
with nothing reported since, the system is asked with poll(). Counted
per socket, the count was set back by any wait, even one woken by its
timer with nothing reported, and by any event of the socket, even for
the other direction. sockmap(), finding nothing to do after an answer,
sleeps in such a wait: the readiness found not to be there, a splice()
into a full pipe given the socket's, was trusted again after each
sleep, until it gave up after 11 turns with 93. Under load every third
session of a relay or a CONNECT ended so in the middle of its data.

What was answered and not reported since is now kept per bit: an event
makes its own bits fresh, a wait answers with what it found, and those
are asked again next time.

poll() of the BSDs and macOS tells a socket we shut down for sending
as hung up, which the loop does not (see loop.c): asking it, that is
dropped, or httpsrv lingering for the client's end found it hung up
and not readable, and turned until its time was out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/mainloop.c

@@ -55,7 +55,8 @@ struct sockreg {
 	struct loopio io;	/* edge-triggered, for reading and writing */
 	short ready;		/* reported ready for, not found not to be since */
 	short want;		/* what the fiber waiting on it waits for */
-	unsigned char spins;	/* waits answered at once since it was reported */
+	short stale;		/* of ready, what a wait was answered with and
+				   nothing reported since */
 	struct fiberwait *w;	/* the wait of the fiber waiting on it */
 };
 
@@ -214,7 +215,7 @@ static void sockregio(struct loop *l, struct loopio *io, short revents){
 	struct fiberwait *w = r->w;
 
 	r->ready |= revents;
-	r->spins = 0;
+	r->stale &= ~revents;
 	/* nothing of r is touched after the fiber runs: it may close it */
 	if(w && !w->woken && (revents & (r->want | POLLERR | POLLHUP))){
 		w->woken = 1;
@@ -257,7 +258,7 @@ static void fiberblocked(SOCKET s, short events){
 
 	if(lp && (unsigned)s < lp->nregs && lp->regs[s]){
 		lp->regs[s]->ready &= ~events;
-		lp->regs[s]->spins = 0;
+		lp->regs[s]->stale &= ~events;
 	}
 }
 
@@ -300,21 +301,27 @@ static int fiberpollregs(struct looper *lp, struct pollfd *fds, unsigned nfds, i
 			goto DONE;
 		}
 		have = regs[i]->ready & (want | POLLERR | POLLHUP);
-		if(have && regs[i]->spins){
+		if(have & regs[i]->stale){
 			struct pollfd p;
 
 			p.fd = fds[i].fd;
 			p.events = POLLIN | POLLOUT;
 			p.revents = 0;
 			if(poll(&p, 1, 0) >= 0) regs[i]->ready = p.revents & READYBITS;
-			regs[i]->spins = 0;
+#ifdef LOOP_KQUEUE
+			/* poll() there tells a socket shut down for sending, by us,
+			   as hung up, the loop does not: its end of reading is
+			   readable */
+			regs[i]->ready &= ~POLLHUP;
+#endif
+			regs[i]->stale = 0;
 			have = regs[i]->ready & (want | POLLERR | POLLHUP);
 		}
 		fds[i].revents = have;
 		if(have) res++;
 	}
 	if(res){
-		for(i = 0; i < nfds; i++) if(fds[i].revents) regs[i]->spins++;
+		for(i = 0; i < nfds; i++) if(regs[i]) regs[i]->stale |= fds[i].revents;
 		goto DONE;
 	}
 	memset(&w, 0, sizeof(w));
@@ -333,8 +340,8 @@ static int fiberpollregs(struct looper *lp, struct pollfd *fds, unsigned nfds, i
 	if(timeout > 0) looptimerdel(&lp->l, &w.t);
 	for(i = 0; i < nfds; i++) if(regs[i]){
 		regs[i]->w = NULL;
-		regs[i]->spins = 0;
 		fds[i].revents = regs[i]->ready & (fds[i].events & (POLLIN | POLLOUT) | POLLERR | POLLHUP);
+		regs[i]->stale |= fds[i].revents;
 		if(fds[i].revents) res++;
 	}
 DONE: