Name the HTTPSRV service in the string table

S_HTTPSRV, the type of admin and httpsrv services, was added before
S_ZOMBIE without a name in the string table, so its index held
"ZOMBIE": the admin services page, %N in logs and RADIUS named admin
and httpsrv services and their clients ZOMBIE, and a zombie service had
no name at all (NULL). Index 21 is now "HTTPSRV", 22 "ZOMBIE"; 22 was
unused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.github/workflows/build-watcom.yml

@@ -40,6 +40,8 @@ jobs:
       run: |
        echo "volatile char VerSion[]=^"3APA3A-3proxy-Internal-Build: 3proxy-%RELEASE%-%NOW%\r\n^";" >>src/3proxy.c
        nmake /F Makefile.watcom
+       if errorlevel 1 exit /b 1
+       if not exist bin\3proxy.exe (echo bin\3proxy.exe was not built & exit /b 1)
     - name: make dist dir
       shell: cmd
       run: |

CMakeLists.txt

@@ -63,7 +63,7 @@ option(3PROXY_USE_OPENSSL "Enable TLS/SSL support via OpenSSL (fallback when wol
 option(3PROXY_USE_PCRE2 "Enable PCRE2 regex filtering" ON)
 option(3PROXY_USE_PAM "Enable PAM/PamAuth" ON)
 option(3PROXY_USE_ODBC "Enable ODBC support (Unix only, always ON on Windows)" OFF)
-option(3PROXY_USE_SPLICE "Build Linux splice() support, slower than read/write for most traffic (Linux only)" OFF)
+option(3PROXY_USE_SPLICE "Build Linux splice() support, used by services given -s (Linux only)" ON)
 option(3PROXY_USE_POLL "Use poll() instead of select() (Unix only)" ON)
 option(3PROXY_USE_WSAPOLL "Use WSAPoll instead of select() (Windows only)" ON)
 option(3PROXY_USE_NETFILTER "Enable Linux netfilter support (Linux only)" ON)
@@ -415,11 +415,12 @@ set(3PROXY_CORE_SOURCES
 # Object libraries for common sources (shared between executables)
 # ============================================================================
 
-# Common object library (sockmap, sockgetchar, common, log)
+# Common object library (sockmap, sockgetchar, common, sem, log)
 add_library(common_obj OBJECT
     src/sockmap.c
     src/sockgetchar.c
     src/common.c
+    src/sem.c
     src/log.c
 )
 target_include_directories(common_obj PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
@@ -728,12 +729,12 @@ foreach(PROXY_NAME proxy socks pop3p imapp smtpp ftppr tcppm udppm tlspr)
         ${CMAKE_CURRENT_SOURCE_DIR}/src
     )
 
-    target_compile_definitions(${PROXY_NAME} PRIVATE
+    target_compile_definitions(${PROXY_NAME} PRIVATE WITHMAIN)
-        WITHMAIN
-        NOPORTMAP
-    )
 
-    if(NOT PROXY_NAME STREQUAL "udppm")
+    # each takes the main loop of its kind only: udppm UDP, the others TCP
+    if(PROXY_NAME STREQUAL "udppm")
+        target_compile_definitions(${PROXY_NAME} PRIVATE NOTCPMAIN)
+    else()
         target_compile_definitions(${PROXY_NAME} PRIVATE NOUDPMAIN)
     endif()
 
@@ -743,8 +744,12 @@ foreach(PROXY_NAME proxy socks pop3p imapp smtpp ftppr tcppm udppm tlspr)
         target_compile_definitions(${PROXY_NAME} PRIVATE ANONYMOUS)
     endif()
 
+    # port mappers take port, host and port as arguments, the others options
+    # only; one of the two is defined, as in src/Makefile.inc
     if(PROXY_NAME STREQUAL "tcppm" OR PROXY_NAME STREQUAL "udppm" OR PROXY_NAME STREQUAL "tlspr")
         target_compile_definitions(${PROXY_NAME} PRIVATE PORTMAP)
+    else()
+        target_compile_definitions(${PROXY_NAME} PRIVATE NOPORTMAP)
     endif()
 
     if(WIN32)

Makefile.Linux

@@ -4,9 +4,9 @@
 # add -DWITH_ODBC to CFLAGS and -lodbc to LIBS to compile with ODBC
 # library support. Add -DSAFESQL for poorely written ODBC library / drivers.
 #
-# Linux splice() support is not built by default, it is slower than the
+# Linux splice() support is built, but used only by services given the -s
-# read/write path for most traffic. Add -DWITHSPLICE to CFLAGS to build it,
+# option: it is faster for large transfers and slower for small requests.
-# it must additionally be enabled per service with the -s option.
+# Remove -DWITHSPLICE from CFLAGS to build without it.
 
 BUILDDIR = ../bin/
 PREFIX ?= 3proxy_
@@ -14,7 +14,7 @@ CRYPT_PREFIX ?= $(PREFIX)
 CC ?= gcc
 
 CFLAGS ?= -O3 -flto
-CFLAGS += -fno-strict-aliasing -c -pthread -D_GNU_SOURCE -DGETHOSTBYNAME_R -D_THREAD_SAFE -D_REENTRANT -DFD_SETSIZE=4096 -DWITH_POLL -DWITH_NETFILTER -D WITH_UN
+CFLAGS += -fno-strict-aliasing -c -pthread -DWITHSPLICE -D_GNU_SOURCE -DGETHOSTBYNAME_R -D_THREAD_SAFE -D_REENTRANT -DFD_SETSIZE=4096 -DWITH_POLL -DWITH_NETFILTER -D WITH_UN
 COUT = -o 
 LN ?= ${CC}
 DCFLAGS ?= -fPIC

doc/html/highload.html

@@ -34,7 +34,8 @@ Each connected client requires:
 <ul>
 <li>1 thread (process)
 <li>2 sockets (file descriptors). For FTP, 4 sockets are required.
-<br>Up to 128K of kernel buffer memory. This is the theoretical maximum; actual numbers depend on connection quality and traffic amount.
+<br>With splice() (Linux, -s), a pipe (2 file descriptors) for each spliced direction: up to 4 more file descriptors, 6 for FTP.
+<br>Up to 128K (up to 256K with splice()) of kernel buffer memory. This is the theoretical maximum; actual numbers depend on connection quality and traffic amount.
 <br>1 additional socket (file descriptor) during name resolution for non-cached names
 <br>1 additional socket during authentication or logging for RADIUS authentication or logging.
 <li>1 ephemeral port (3 ephemeral ports for FTP connections).
@@ -279,6 +280,8 @@ tuning anything else.
 
 If 3proxy is used in a VPS environment, there can be additional limitations.
 For example, kernel resources, system CPU usage, and IOCTLs can be limited differently, and this can become a bottleneck.
+splice() (see below) moves work into the kernel, so where the hosting limits kernel resources
+(seen as nearly 100% CPU usage without any real activity of the application) do not enable it.
 
 <h4>Extending the Ephemeral Port Range</h4>
 
@@ -641,17 +644,42 @@ monitor "/some/path/3proxy/3proxy.ver"
 e.g., with "touch /some/path/3proxy/3proxy.ver".
 </ol>
 
-<h4>Use TCP_NODELAY to Speed Up Connections with Small Amounts of Data</h4>
+<h4>TCP_NODELAY for Connections with Small Amounts of Data</h4>
 
-If most requests require an exchange with a small amount of data in both directions
+TCP_NODELAY is set by default on client (oc) and server (os)
-without the need for bandwidth, e.g., messengers or small web requests,
+connections: without it, Nagle's algorithm holds back a short write until the
-you can eliminate Nagle's algorithm delay with the TCP_NODELAY flag. Usage example:
+previous one is acknowledged, and a peer delays its acknowledgement, so a reply
+written in more than one part (e.g. HTTP headers, then a short body) waits tens
+of milliseconds. On slow connections with high delays, where the bandwidth is the
+bottleneck and fewer packets matter more than latency, it can be turned off:
 <pre>
-proxy -osTCP_NODELAY -ocTCP_NODELAY
+proxy -osTCP_NODELAY=0 -ocTCP_NODELAY=0
 </pre>
-sets TCP_NODELAY for client (oc) and server (os) connections.
+
-<p>Do not use TCP_NODELAY on slow connections with high delays when
+<h4>Use Splice to Speed Up Large Data Transfers</h4>
-connection bandwidth is a bottleneck.
+
+On Linux, splice() passes data between the client and the server connections
+through a kernel pipe, without copying it into the 3proxy process. Linux builds
+include it, but a service uses it only when given the -s option:
+<pre>
+proxy -s
+socks -s
+tcppm -s3 8080 192.168.1.1 80
+</pre>
+-s alone splices both directions where it pays: an HTTP body of known length if at
+least 16K of it is left to read, a stream (CONNECT, SOCKS, tcppm) once it carries
+large reads, going back to copying when they turn small. -s1 always splices what the
+server sends to the client (downloads), -s2 what the client sends to the server
+(uploads), -s3 both; -s5, -s6 and -s7 (the same as -s) do it for those directions
+only where it pays. -s0, the default, turns it off. In builds without it (CMake option 3PROXY_USE_SPLICE=OFF,
+or -DWITHSPLICE removed from the Makefile) -s is accepted and ignored.
+<p>splice() pays off when most of the traffic is large transfers: in the
+<a href="https://github.com/3proxy/3proxy/blob/devel/bench/RESULTS.md">benchmarks</a> it moved
+1.3 to 2.1 times more data than the read/write path at 40-55% less CPU per gigabyte.
+It does not help small requests and short connections, which go slower with it, and
+every spliced direction needs a pipe (two file descriptors) and more kernel
+buffer memory. Data filters are not applied to spliced data.
+<p>TCP_NODELAY and splice() do not conflict and can be combined on high-speed connections.
 
 <h4>Add Grace Delay to Reduce System Calls</h4>
 
@@ -661,7 +689,7 @@ the minimal number of packets in the same direction to apply the algorithm,
 and the last value is the delay added after polling and prior to reading data.
 The example above adds a 10-millisecond delay before reading data if the average
 polling size is below 8000 bytes and 3 read operations have been made in the same
-direction. <pre>logdump 1 1</pre> is useful
+direction. It's especially useful with splice. <pre>logdump 1 1</pre> is useful
 to see how grace delays work; choose a delay value to avoid filling the read
 buffer (typically 64K) but keep the request sizes close to the chosen average
 on large file uploads/downloads.

doc/html/man5/3proxy.cfg.5.html

@@ -210,18 +210,30 @@ or hostname, useful in case of dynamic DNS. <b><br>
 -oR</b><i>OPTIONS</i> options for proxy-to-client
 (<b>-oc</b>), proxy-to-server (<b>-os</b>), proxy listening
 (<b>-ol</b>), connect back client (<b>-or</b>), connect back
-listening (<b>-oR</b>) sockets. Options like TCP_CORK,
+listening (<b>-oR</b>) sockets. <i>OPTIONS</i> is a comma
-TCP_NODELAY, TCP_DEFER_ACCEPT, TCP_QUICKACK, TCP_TIMESTAMPS,
+separated list of <i>NAME</i> or <i>NAME</i>=<i>VALUE</i>; a
+name alone sets the option to 1 (TCP_MAXSEG to the value of
+<b>maxseg</b>), and <i>NAME</i>=0 turns it off. Options add
+to the defaults and to the options given earlier for the
+same socket. An unknown name or a value which is not a
+number is an error. TCP_NODELAY is set by default on
+proxy-to-client and proxy-to-server sockets, so a reply
+written in several parts is not delayed by Nagle&rsquo;s
+algorithm; use <b>-ocTCP_NODELAY=0</b> or
+<b>-osTCP_NODELAY=0</b> to turn it off, e.g. to send fewer
+packets on slow links. Options like TCP_CORK, TCP_NODELAY,
+TCP_DEFER_ACCEPT, TCP_QUICKACK, TCP_TIMESTAMPS,
 TCP_FASTOPEN, SO_REUSEADDR, SO_REUSEPORT,
 SO_EXCLUSIVEADDRUSE, SO_PORT_SCALABILITY,
 SO_REUSE_UNICASTPORT, SO_KEEPALIVE, SO_DONTROUTE may be
 supported depending on OS. SO_REUSEADDR and SO_REUSEPORT are
-set on the listening socket by default on Unix. On Windows
+set by default on the listening sockets (<b>-ol</b>,
-SO_REUSEADDR is not set: it is not required to rebind a
+<b>-oR</b>) on Unix; <b>-olSO_REUSEPORT=0</b> turns one off.
-listening port and it only lets another local process bind
+On Windows SO_REUSEADDR is not set: it is not required to
-the same address and port. Use SO_EXCLUSIVEADDRUSE (Windows)
+rebind a listening port and it only lets another local
-on the listening socket (<b>-ol</b>) to prevent that.
+process bind the same address and port. Use
-<b><br>
+SO_EXCLUSIVEADDRUSE (Windows) on the listening socket
+(<b>-ol</b>) to prevent that. <b><br>
 -H</b> (for all services) Expect HAProxy PROXY protocol v1
 header on incoming connection. This allows the proxy to
 receive real client IP address from HAProxy or other load
@@ -236,12 +248,23 @@ second. Useful to minimize polling <b><br>
 -s</b> <br>
 (for admin) secure, allow only secure operations, currently
 only traffic counters view without ability to reset. <br>
-(for TCP services, Linux) enable splice(). splice() is not
+(for TCP services, Linux) use splice() to pass data between
-built by default and is disabled even when built, because
+the client and the server without copying it through 3proxy.
-current Linux does not implement SPLICE_F_MOVE, so no real
+<b>-s</b> alone splices both directions where it pays: a
-zero-copy takes place and the read/write path is faster for
+body of known length if at least 16K of it is left to read,
-most traffic. Rebuild with -DWITHSPLICE to make -s
+a stream once it carries large reads, back to copying when
-available, -s0 disables it explicitly. <b><br>
+they turn small. <b>-s1</b> always splices what the server
+sends to the client (downloads), <b>-s2</b> what the client
+sends (uploads), <b>-s3</b> both; adding 4 (<b>-s5</b>,
+<b>-s6</b>, <b>-s7</b>, the last the same as <b>-s</b>)
+splices those directions only where it pays. <b>-s0</b>, the
+default, turns it off. splice() suits large transfers: they
+go faster and use less CPU; small requests go slower with
+it. A spliced direction needs a pipe (two file descriptors),
+made when first needed and kept for the connection, and more
+kernel memory; data filters are not applied to spliced data.
+Available in builds with WITHSPLICE, the default on Linux.
+<b><br>
 -C</b> <br>
 (for TCP services) a connection closed by any of the sides
 terminates the session, buffered data is delivered before
@@ -277,7 +300,12 @@ client reported) <b><br>
 -a1</b> (for proxy) anonymous proxy (random client
 information reported) <b><br>
 -a2</b> (for proxy) generate Via: and X-Forwarded-For:
-instead of Forwarded: <br>
+instead of Forwarded: <b><br>
+-Xftp</b> (for proxy) fetch <b>ftp://</b> URLs, in a build
+with FTP support. Without this the scheme is not one the
+service knows, whatever the build: fetching it opens a
+second connection and builds a listing, which a service
+serving http has no use for. <br>
 Also, all options mentioned for <b>proxy</b>(8)
 <b>socks</b>(8) <b>pop3p</b>(8) <b>imapp</b>(8)
 <b>tcppm</b>(8) <b>udppm</b>(8) <b>ftppr</b>(8) <br>
@@ -1300,8 +1328,15 @@ groupname. Unix only.</p>
 <i>&lt;path&gt;</i> [<i>&lt;uid&gt;</i>]
 [<i>&lt;gid&gt;</i>] <br>
 calls chroot(path) and sets gid/uid. Unix only. uid/gid
-supported since 0.9, can be numeric or
+supported since 0.9, can be numeric or username/groupname.
-username/groupname</p>
+The time zone and the syslog connection are set up before
+the call. Plugins which read their own configuration or load
+modules while running need them inside the chroot: PAM
+(<i>/etc/pam.d</i> and the modules it names, read for every
+authentication), LDAP (<i>ldap.conf</i> and, for ldaps, the
+CA certificates), ODBC (<i>odbc.ini</i>, <i>odbcinst.ini</i>
+and the driver libraries, loaded again whenever the
+connection is reopened).</p>
 
 
 <p style="margin-left:9%; margin-top: 1em"><b>stacksize</b>
@@ -1590,10 +1625,12 @@ authentication and/or allow/deny ACLs.</p>
 and FTP are built when they are asked for, and are not in a
 default build. <b>MAILPROXY=true</b> builds <b>pop3p</b>,
 <b>imapp</b> and <b>smtpp</b>, and <b>FTP=true</b> builds
-<b>ftppr</b> and the <b>ftp://</b> scheme of the HTTP proxy;
+<b>ftppr</b> and the <b>ftp://</b> scheme of the HTTP proxy,
-with CMake the switches are <b>-D3PROXY_USE_MAILPROXY=ON</b>
+which a <b>proxy</b> service then still has to ask for with
-and <b>-D3PROXY_USE_FTP=ON</b>. The standalone binaries of
+<b>-Xftp</b>; with CMake the switches are
-those services are built with them and not without. <br>
+<b>-D3PROXY_USE_MAILPROXY=ON</b> and
+<b>-D3PROXY_USE_FTP=ON</b>. The standalone binaries of those
+services are built with them and not without. <br>
 A configuration naming a service which was not built is
 still read. The three mail proxies become <b>tlspr</b>
 negotiating STARTTLS in that protocol, which is what most of

man/3proxy.cfg.5

@@ -184,8 +184,10 @@ connect to given remote HOST:port instead of listening local connection on -p or
 .br
 .B -oc\fIOPTIONS\fB, -os\fIOPTIONS\fB, -ol\fIOPTIONS\fB, -or\fIOPTIONS\fB, -oR\fIOPTIONS\fR
 options for proxy-to-client (\fB-oc\fR), proxy-to-server (\fB-os\fR), proxy listening (\fB-ol\fR), connect back client (\fB-or\fR), connect back listening (\fB-oR\fR) sockets.
+\fIOPTIONS\fR is a comma separated list of \fINAME\fR or \fINAME\fR=\fIVALUE\fR; a name alone sets the option to 1 (TCP_MAXSEG to the value of \fBmaxseg\fR), and \fINAME\fR=0 turns it off. Options add to the defaults and to the options given earlier for the same socket. An unknown name or a value which is not a number is an error.
+TCP_NODELAY is set by default on proxy-to-client and proxy-to-server sockets, so a reply written in several parts is not delayed by Nagle's algorithm; use \fB-ocTCP_NODELAY=0\fR or \fB-osTCP_NODELAY=0\fR to turn it off, e.g. to send fewer packets on slow links.
 Options like TCP_CORK, TCP_NODELAY, TCP_DEFER_ACCEPT, TCP_QUICKACK, TCP_TIMESTAMPS, TCP_FASTOPEN, SO_REUSEADDR, SO_REUSEPORT, SO_EXCLUSIVEADDRUSE, SO_PORT_SCALABILITY, SO_REUSE_UNICASTPORT, SO_KEEPALIVE, SO_DONTROUTE may be supported depending on OS.
-SO_REUSEADDR and SO_REUSEPORT are set on the listening socket by default on Unix. On Windows SO_REUSEADDR is not set: it is not required to rebind a listening port and it only lets another local process bind the same address and port. Use SO_EXCLUSIVEADDRUSE (Windows) on the listening socket (\fB-ol\fR) to prevent that.
+SO_REUSEADDR and SO_REUSEPORT are set by default on the listening sockets (\fB-ol\fR, \fB-oR\fR) on Unix; \fB-olSO_REUSEPORT=0\fR turns one off. On Windows SO_REUSEADDR is not set: it is not required to rebind a listening port and it only lets another local process bind the same address and port. Use SO_EXCLUSIVEADDRUSE (Windows) on the listening socket (\fB-ol\fR) to prevent that.
 .br
 .B -H
 (for all services) Expect HAProxy PROXY protocol v1 header on incoming connection.
@@ -200,11 +202,18 @@ delay GRACE_DELAY milliseconds before polling if average polling size is below G
  (for admin) secure, allow only secure operations, currently only traffic counters
 view without ability to reset.
 .br
- (for TCP services, Linux) enable splice(). splice() is not built by default and
+ (for TCP services, Linux) use splice() to pass data between the client and the
-is disabled even when built, because current Linux does not implement
+server without copying it through 3proxy. \fB-s\fR alone splices both directions where
-SPLICE_F_MOVE, so no real zero-copy takes place and the read/write path is faster
+it pays: a body of known length if at least 16K of it is left to read, a stream
-for most traffic. Rebuild with -DWITHSPLICE to make -s available, -s0 disables it
+once it carries large reads, back to copying when they turn small. \fB-s1\fR always
-explicitly.
+splices what the server sends to the client (downloads), \fB-s2\fR what the client
+sends (uploads), \fB-s3\fR both; adding 4 (\fB-s5\fR, \fB-s6\fR, \fB-s7\fR, the last the same as
+\fB-s\fR) splices those directions only where it pays. \fB-s0\fR, the default, turns it
+off. splice() suits large transfers: they go faster and use less CPU; small
+requests go slower with it. A spliced direction needs a pipe (two file
+descriptors), made when first needed and kept for the connection, and more
+kernel memory; data filters are not applied to spliced data. Available in builds
+with WITHSPLICE, the default on Linux.
 .br
 .B -C
  (for TCP services) a connection closed by any of the sides terminates the
@@ -244,6 +253,12 @@ Never ask for username/password
 .br
 .B -a2
 (for proxy) generate Via: and X-Forwarded-For: instead of Forwarded:
+.br
+.B -Xftp
+(for proxy) fetch \fBftp://\fR URLs, in a build with FTP support. Without
+this the scheme is not one the service knows, whatever the build: fetching it
+opens a second connection and builds a listing, which a service serving http
+has no use for.
 .br
  Also, all options mentioned for 
 .BR proxy (8)
@@ -1373,7 +1388,12 @@ for all threads.
 .BR chroot
 \fI<path>\fR [\fI<uid>\fR] [\fI<gid>\fR]
 .br
- calls chroot(path) and sets gid/uid. Unix only. uid/gid supported since 0.9, can be numeric or username/groupname
+ calls chroot(path) and sets gid/uid. Unix only. uid/gid supported since 0.9, can be numeric or username/groupname.
+The time zone and the syslog connection are set up before the call.
+Plugins which read their own configuration or load modules while running need them inside the chroot:
+PAM (\fI/etc/pam.d\fR and the modules it names, read for every authentication),
+LDAP (\fIldap.conf\fR and, for ldaps, the CA certificates),
+ODBC (\fIodbc.ini\fR, \fIodbcinst.ini\fR and the driver libraries, loaded again whenever the connection is reopened).
 
 .br
 .BR stacksize
@@ -1639,7 +1659,8 @@ authentication and/or allow/deny ACLs.
 The mail proxies and FTP are built when they are asked for, and are not in a
 default build. \fBMAILPROXY=true\fR builds \fBpop3p\fR, \fBimapp\fR and
 \fBsmtpp\fR, and \fBFTP=true\fR builds \fBftppr\fR and the \fBftp://\fR
-scheme of the HTTP proxy; with CMake the switches are
+scheme of the HTTP proxy, which a \fBproxy\fR service then still has to ask
+for with \fB-Xftp\fR; with CMake the switches are
 \fB-D3PROXY_USE_MAILPROXY=ON\fR and \fB-D3PROXY_USE_FTP=ON\fR. The standalone
 binaries of those services are built with them and not without.
 .br

src/3proxy.c

@@ -74,13 +74,8 @@ void __stdcall CommandHandler( DWORD dwCommand )
         SetStatus( SERVICE_STOP_PENDING, 0, 1 );
 	conf.timetoexit = 1;
 	conf.paused++;
-	Sleep(2000);
+	/* ServiceMain() reports the service stopped once cyclestep() has
-        SetStatus( SERVICE_STOPPED, 0, 0 );
+	   seen this and the services are done */
-#ifdef WITH_ODBC
-	_3proxy_mutex_lock(&log_mutex);
-	close_sql();
-	_3proxy_mutex_unlock(&log_mutex);
-#endif
         break;
     case SERVICE_CONTROL_PAUSE:
         SetStatus( SERVICE_PAUSE_PENDING, 0, 1 );
@@ -106,6 +101,13 @@ void __stdcall ServiceMain(int argc, unsigned char* argv[] )
     SetStatus( SERVICE_START_PENDING, 0, 1 );
     SetStatus( SERVICE_RUNNING, 0, 0 );
     cyclestep();
+#ifdef WITH_ODBC
+    _3proxy_mutex_lock(&log_mutex);
+    close_sql();
+    _3proxy_mutex_unlock(&log_mutex);
+#endif
+    /* the last thing: the process may be ended once this is reported */
+    SetStatus( SERVICE_STOPPED, 0, 0 );
 }
 #endif
 
@@ -240,6 +242,39 @@ void dumpcounters(struct trafcount *tlin, int counterd){
  }
 }
 
+/* Lets the services finish before the process exits, for EXITWAIT ms at
+   most: each one waits for its connections, and a connection may hang on
+   its peer. Posts left over from reloads are consumed on the way, the
+   count decides. */
+#define EXITWAIT 3000
+static unsigned long nowms(void){
+#ifdef _WIN32
+ return (unsigned long)GetTickCount();
+#else
+ struct timeval tv;
+
+ gettimeofday(&tv, NULL);
+ return (unsigned long)tv.tv_sec * 1000 + tv.tv_usec / 1000;
+#endif
+}
+
+static void waitservices(void){
+ unsigned long start = nowms(), passed;
+
+ for(;;){
+	int n;
+
+	_3proxy_mutex_lock(&runningservices_mutex);
+	n = runningservices;
+	_3proxy_mutex_unlock(&runningservices_mutex);
+	if(!n) return;
+	/* a clock set back makes this huge, which ends the wait as well */
+	passed = nowms() - start;
+	if(passed >= EXITWAIT) return;
+	if(_3proxy_sem_timedlock(serviceexited, (unsigned)(EXITWAIT - passed))) return;
+ }
+}
+
 void cyclestep(void){
  struct tm *tm;
  time_t minutecounter;
@@ -336,9 +371,7 @@ void cyclestep(void){
 	if(conf.timetoexit){
 		conf.paused++;
 		doschedule();
-		usleep(SLEEPTIME*999);
+		waitservices();
-		usleep(SLEEPTIME*999);
-		usleep(SLEEPTIME*999);
 		return;
 	}
 		
@@ -517,6 +550,7 @@ int WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLine, int
   }
 
   _3proxy_mutex_init(&config_mutex);
+  _3proxy_mutex_init(&runningservices_mutex);
   _3proxy_mutex_init(&bandlim_mutex);
   _3proxy_mutex_init(&connlim_mutex);
   _3proxy_mutex_init(&tc_mutex);
@@ -524,7 +558,7 @@ int WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLine, int
 #ifndef NORADIUS
   _3proxy_mutex_init(&rad_mutex);
 #endif
-  if(_3proxy_sem_init(conf.threadinit, 1, 1)){
+  if(_3proxy_sem_init_timed(serviceexited, 0, 0x7fffffff)){
     fprintf(stderr, "semaphore init failed\n");
     return 1;
   }
@@ -547,7 +581,9 @@ int WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLine, int
   res = readconfig(fp);
   conf.version++;
 
-  if(res) RETURN(res);
+  /* readconfig() names the failing line, which an exit status would cut
+     to 8 bits: a failed configuration exits with 1 */
+  if(res) RETURN(1);
   fclose(fp); fp = NULL;
 
 #ifdef _WIN32
@@ -580,6 +616,6 @@ int WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLine, int
 CLEARRETURN:
 
  if(fp && fp != stdin) {fclose(fp); fp = NULL;}
- return 0;
+ return res;
 
 }

src/Makefile.inc

@@ -13,6 +13,9 @@ udpsockmap$(OBJSUFFICS): udpsockmap.c proxy.h structures.h
 common$(OBJSUFFICS): common.c proxy.h structures.h
 	$(CC) $(CFLAGS) common.c
 
+sem$(OBJSUFFICS): sem.c proxy.h structures.h
+	$(CC) $(CFLAGS) sem.c
+
 plugins$(OBJSUFFICS): plugins.c proxy.h structures.h
 	$(CC) $(CFLAGS) plugins.c
 
@@ -44,7 +47,7 @@ tcppm$(OBJSUFFICS): tcppm.c proxy.h structures.h proxymain.c
 	$(CC) $(CFLAGS) $(DEFINEOPTION)WITHMAIN $(DEFINEOPTION)PORTMAP $(DEFINEOPTION)NOUDPMAIN tcppm.c
 
 udppm$(OBJSUFFICS): udppm.c proxy.h structures.h proxymain.c
-	$(CC) $(CFLAGS) $(DEFINEOPTION)WITHMAIN $(DEFINEOPTION)PORTMAP udppm.c
+	$(CC) $(CFLAGS) $(DEFINEOPTION)WITHMAIN $(DEFINEOPTION)PORTMAP $(DEFINEOPTION)NOTCPMAIN udppm.c
 
 tlspr$(OBJSUFFICS): tlspr.c proxy.h structures.h proxymain.c
 	$(CC) $(CFLAGS) $(DEFINEOPTION)WITHMAIN $(DEFINEOPTION)PORTMAP $(DEFINEOPTION)NOUDPMAIN tlspr.c
@@ -56,32 +59,32 @@ socks$(OBJSUFFICS): socks.c proxy.h structures.h proxymain.c
 3proxy$(OBJSUFFICS): 3proxy.c proxy.h structures.h
 	$(CC) $(CFLAGS) 3proxy.c
 
-$(BUILDDIR)$(PREFIX)proxy$(EXESUFFICS): sockmap$(OBJSUFFICS) proxy$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS) base64$(OBJSUFFICS) ftp$(OBJSUFFICS) $(COMPATLIBS)
+$(BUILDDIR)$(PREFIX)proxy$(EXESUFFICS): sockmap$(OBJSUFFICS) proxy$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS) base64$(OBJSUFFICS) ftp$(OBJSUFFICS) $(COMPATLIBS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)proxy$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) proxy$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) base64$(OBJSUFFICS) ftp$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)proxy$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) proxy$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) base64$(OBJSUFFICS) ftp$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
 
-$(BUILDDIR)$(PREFIX)pop3p$(EXESUFFICS): sockmap$(OBJSUFFICS) pop3p$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS) $(COMPATLIBS)
+$(BUILDDIR)$(PREFIX)pop3p$(EXESUFFICS): sockmap$(OBJSUFFICS) pop3p$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS) $(COMPATLIBS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)pop3p$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) pop3p$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)pop3p$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) pop3p$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
 
-$(BUILDDIR)$(PREFIX)imapp$(EXESUFFICS): sockmap$(OBJSUFFICS) imapp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS) base64$(OBJSUFFICS) $(COMPATLIBS)
+$(BUILDDIR)$(PREFIX)imapp$(EXESUFFICS): sockmap$(OBJSUFFICS) imapp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS) base64$(OBJSUFFICS) $(COMPATLIBS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)imapp$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) imapp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) base64$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)imapp$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) imapp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) base64$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
 
-$(BUILDDIR)$(PREFIX)smtpp$(EXESUFFICS): sockmap$(OBJSUFFICS) smtpp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS) base64$(OBJSUFFICS) $(COMPATLIBS)
+$(BUILDDIR)$(PREFIX)smtpp$(EXESUFFICS): sockmap$(OBJSUFFICS) smtpp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS) base64$(OBJSUFFICS) $(COMPATLIBS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)smtpp$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) smtpp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) base64$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)smtpp$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) smtpp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) base64$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
 
-$(BUILDDIR)$(PREFIX)ftppr$(EXESUFFICS): sockmap$(OBJSUFFICS) ftppr$(OBJSUFFICS) ftp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS) $(COMPATLIBS)
+$(BUILDDIR)$(PREFIX)ftppr$(EXESUFFICS): sockmap$(OBJSUFFICS) ftppr$(OBJSUFFICS) ftp$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS) $(COMPATLIBS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)ftppr$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) ftppr$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS) ftp$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)ftppr$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) ftppr$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS) ftp$(OBJSUFFICS) $(COMPATLIBS) $(LIBS)
 
-$(BUILDDIR)$(PREFIX)socks$(EXESUFFICS): sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) socks$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS)
+$(BUILDDIR)$(PREFIX)socks$(EXESUFFICS): sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) socks$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)socks$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) socks$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)socks$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) socks$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) $(LIBS)
 
-$(BUILDDIR)$(PREFIX)tcppm$(EXESUFFICS): sockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) tcppm$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS)
+$(BUILDDIR)$(PREFIX)tcppm$(EXESUFFICS): sockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) tcppm$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)tcppm$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) tcppm$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)tcppm$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) tcppm$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) $(LIBS)
 
-$(BUILDDIR)$(PREFIX)udppm$(EXESUFFICS): sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) udppm$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS) hash$(OBJSUFFICS)
+$(BUILDDIR)$(PREFIX)udppm$(EXESUFFICS): sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) udppm$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS) hash$(OBJSUFFICS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)udppm$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) udppm$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) hash$(OBJSUFFICS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)udppm$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) udppm$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) hash$(OBJSUFFICS) $(LIBS)
 
-$(BUILDDIR)$(PREFIX)tlspr$(EXESUFFICS): sockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) tlspr$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS)
+$(BUILDDIR)$(PREFIX)tlspr$(EXESUFFICS): sockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) tlspr$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS)
-	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)tlspr$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) tlspr$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) $(LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)$(PREFIX)tlspr$(EXESUFFICS) $(LDFLAGS) sockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) tlspr$(OBJSUFFICS) log$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) $(LIBS)
 
 mainfunc$(OBJSUFFICS): proxy.h structures.h proxymain.c
 	$(CC) $(COUT)mainfunc$(OBJSUFFICS) $(CFLAGS) $(DEFINEOPTION)MODULEMAINFUNC=mainfunc proxymain.c
@@ -194,6 +197,6 @@ ssl$(OBJSUFFICS): ssl.c structures.h proxy.h ssl.h
 pcre$(OBJSUFFICS): pcre.c structures.h
 	$(CC) $(COUT)pcre$(OBJSUFFICS) $(CFLAGS) $(DEFINEOPTION)WITH_PCRE pcre.c
 
-$(BUILDDIR)3proxy$(EXESUFFICS): 3proxy$(OBJSUFFICS) mainfunc$(OBJSUFFICS) srvproxy$(OBJSUFFICS) srvpop3p$(OBJSUFFICS) srvimapp$(OBJSUFFICS) srvsmtpp$(OBJSUFFICS) srvftppr$(OBJSUFFICS) srvsocks$(OBJSUFFICS) srvtcppm$(OBJSUFFICS) srvtlspr$(OBJSUFFICS) srvauto$(OBJSUFFICS) srvudppm$(OBJSUFFICS) sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) auth$(OBJSUFFICS) acl$(OBJSUFFICS) limiter$(OBJSUFFICS) redirect$(OBJSUFFICS) authradius$(OBJSUFFICS) hash$(OBJSUFFICS) hashtables$(OBJSUFFICS) resolve$(OBJSUFFICS) sql$(OBJSUFFICS) conf$(OBJSUFFICS) log$(OBJSUFFICS) datatypes$(OBJSUFFICS) md4$(OBJSUFFICS) md5$(OBJSUFFICS) blake2$(OBJSUFFICS) 3proxy_crypt$(OBJSUFFICS) base64$(OBJSUFFICS) ftp$(OBJSUFFICS) stringtable$(OBJSUFFICS) $(HTTPSRV_OBJS) srvdnspr$(OBJSUFFICS) plugins$(OBJSUFFICS) mdhash$(OBJSUFFICS) $(SSL_OBJS) $(PCRE_OBJS) $(TRANSPARENT_OBJS) $(COMPATLIBS) $(VERSIONDEP)
+$(BUILDDIR)3proxy$(EXESUFFICS): 3proxy$(OBJSUFFICS) mainfunc$(OBJSUFFICS) srvproxy$(OBJSUFFICS) srvpop3p$(OBJSUFFICS) srvimapp$(OBJSUFFICS) srvsmtpp$(OBJSUFFICS) srvftppr$(OBJSUFFICS) srvsocks$(OBJSUFFICS) srvtcppm$(OBJSUFFICS) srvtlspr$(OBJSUFFICS) srvauto$(OBJSUFFICS) srvudppm$(OBJSUFFICS) sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) auth$(OBJSUFFICS) acl$(OBJSUFFICS) limiter$(OBJSUFFICS) redirect$(OBJSUFFICS) authradius$(OBJSUFFICS) hash$(OBJSUFFICS) hashtables$(OBJSUFFICS) resolve$(OBJSUFFICS) sql$(OBJSUFFICS) conf$(OBJSUFFICS) log$(OBJSUFFICS) datatypes$(OBJSUFFICS) md4$(OBJSUFFICS) md5$(OBJSUFFICS) blake2$(OBJSUFFICS) 3proxy_crypt$(OBJSUFFICS) base64$(OBJSUFFICS) ftp$(OBJSUFFICS) stringtable$(OBJSUFFICS) $(HTTPSRV_OBJS) srvdnspr$(OBJSUFFICS) plugins$(OBJSUFFICS) mdhash$(OBJSUFFICS) $(SSL_OBJS) $(PCRE_OBJS) $(TRANSPARENT_OBJS) $(COMPATLIBS) $(VERSIONDEP)
-	$(LN) $(LNOUT)$(BUILDDIR)3proxy$(EXESUFFICS) $(LDFLAGS) $(VERFILE)  3proxy$(OBJSUFFICS) mainfunc$(OBJSUFFICS) auth$(OBJSUFFICS) acl$(OBJSUFFICS) limiter$(OBJSUFFICS) redirect$(OBJSUFFICS) authradius$(OBJSUFFICS) hash$(OBJSUFFICS) hashtables$(OBJSUFFICS) resolve$(OBJSUFFICS) sql$(OBJSUFFICS) conf$(OBJSUFFICS) datatypes$(OBJSUFFICS) srvauto$(OBJSUFFICS) srvproxy$(OBJSUFFICS) srvpop3p$(OBJSUFFICS) srvimapp$(OBJSUFFICS) srvsmtpp$(OBJSUFFICS) srvftppr$(OBJSUFFICS) srvsocks$(OBJSUFFICS) srvtcppm$(OBJSUFFICS) srvtlspr$(OBJSUFFICS) srvudppm$(OBJSUFFICS) sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) log$(OBJSUFFICS) 3proxy_crypt$(OBJSUFFICS) md4$(OBJSUFFICS) md5$(OBJSUFFICS) blake2$(OBJSUFFICS) base64$(OBJSUFFICS) ftp$(OBJSUFFICS) stringtable$(OBJSUFFICS) $(HTTPSRV_OBJS) srvdnspr$(OBJSUFFICS) plugins$(OBJSUFFICS) mdhash$(OBJSUFFICS) $(SSL_OBJS) $(PCRE_OBJS) $(TRANSPARENT_OBJS) $(COMPATLIBS) $(LIBS) $(PCRE_LIBS)
+	$(LN) $(LNOUT)$(BUILDDIR)3proxy$(EXESUFFICS) $(LDFLAGS) $(VERFILE)  3proxy$(OBJSUFFICS) mainfunc$(OBJSUFFICS) auth$(OBJSUFFICS) acl$(OBJSUFFICS) limiter$(OBJSUFFICS) redirect$(OBJSUFFICS) authradius$(OBJSUFFICS) hash$(OBJSUFFICS) hashtables$(OBJSUFFICS) resolve$(OBJSUFFICS) sql$(OBJSUFFICS) conf$(OBJSUFFICS) datatypes$(OBJSUFFICS) srvauto$(OBJSUFFICS) srvproxy$(OBJSUFFICS) srvpop3p$(OBJSUFFICS) srvimapp$(OBJSUFFICS) srvsmtpp$(OBJSUFFICS) srvftppr$(OBJSUFFICS) srvsocks$(OBJSUFFICS) srvtcppm$(OBJSUFFICS) srvtlspr$(OBJSUFFICS) srvudppm$(OBJSUFFICS) sockmap$(OBJSUFFICS) udpsockmap$(OBJSUFFICS) sockgetchar$(OBJSUFFICS) common$(OBJSUFFICS) sem$(OBJSUFFICS) log$(OBJSUFFICS) 3proxy_crypt$(OBJSUFFICS) md4$(OBJSUFFICS) md5$(OBJSUFFICS) blake2$(OBJSUFFICS) base64$(OBJSUFFICS) ftp$(OBJSUFFICS) stringtable$(OBJSUFFICS) $(HTTPSRV_OBJS) srvdnspr$(OBJSUFFICS) plugins$(OBJSUFFICS) mdhash$(OBJSUFFICS) $(SSL_OBJS) $(PCRE_OBJS) $(TRANSPARENT_OBJS) $(COMPATLIBS) $(LIBS) $(PCRE_LIBS)
 

src/common.c

@@ -187,33 +187,6 @@ size_t threadstacksize(int extra){
 	if(size < (long)PTHREAD_STACK_MIN) size = (long)PTHREAD_STACK_MIN;
 	return (size_t)size;
 }
-
-int _3proxy_sem_init_f(_3proxy_sem_t *sem, unsigned count, unsigned maxcount){
-	sem->count = count;
-	sem->maxcount = maxcount;
-	if(pthread_mutex_init(&sem->mutex, NULL)) return 1;
-	if(pthread_cond_init(&sem->cond, NULL)){
-		pthread_mutex_destroy(&sem->mutex);
-		return 1;
-	}
-	return 0;
-}
-
-void _3proxy_sem_lock_f(_3proxy_sem_t *sem){
-	pthread_mutex_lock(&sem->mutex);
-	while(!sem->count) pthread_cond_wait(&sem->cond, &sem->mutex);
-	sem->count--;
-	pthread_mutex_unlock(&sem->mutex);
-}
-
-void _3proxy_sem_unlock_f(_3proxy_sem_t *sem){
-	pthread_mutex_lock(&sem->mutex);
-	if(sem->count < sem->maxcount){
-		sem->count++;
-		pthread_cond_signal(&sem->cond);
-	}
-	pthread_mutex_unlock(&sem->mutex);
-}
 #endif
 
 struct extparam conf = {
@@ -414,7 +387,6 @@ int
 	return shutdown(s, how);
     }
     int WINAPI def_closesocket(void* state, SOCKET s){
-	usleep(SLEEPTIME);
 	return closesocket(s);
     }
 #else
@@ -742,7 +714,9 @@ int doconnect(struct clientparam * param){
 	if(param->srv->so._getpeername(param->sostate, param->remsock, (struct sockaddr *)&param->sinsr, &size)==-1) {return (14);}
  }
  else {
+#ifndef _WIN32
 	struct linger lg = {1,conf.timeouts[LINGER_TO]};
+#endif
 	int tries = RANGETRIES;
 
 	if(SAISNULL(&param->sinsr)){
@@ -772,9 +746,12 @@ int doconnect(struct clientparam * param){
 			param->sinsl = param->srv->extsa;
 	}
 	*SAPORT(&param->sinsl) = 0;
-	setopts(param->remsock, param->srv->srvsockopts);
+	setopts(param->remsock, &param->srv->srvsockopts);
 
+#ifndef _WIN32
+	/* not on Windows, see the accepting side in proxymain.c */
 	param->srv->so._setsockopt(param->sostate, param->remsock, SOL_SOCKET, SO_LINGER, (char *)&lg, sizeof(lg));
+#endif
 	if(extbindtodevice(param, param->remsock)) return 12;
 #ifdef WITH_UN
 	if(*SAFAMILY(&param->sinsl) != AF_UNIX)

src/conf.c

@@ -38,14 +38,18 @@ _3proxy_mutex_t bandlim_mutex;
 _3proxy_mutex_t connlim_mutex;
 _3proxy_mutex_t tc_mutex;
 _3proxy_mutex_t config_mutex;
+int runningservices = 0;
+_3proxy_mutex_t runningservices_mutex;
+_3proxy_sem_t serviceexited;
 
-int haveerror = 0;
 int linenum = 0;
 
 struct counter_header cheader = {"3CF", (time_t)0};
 struct counter_record crecord;
 
-int mainfunc (int argc, char** argv);
+int mainprepare(int argc, char** argv, void **service);
+int mainrun(void *service);
+void maindrop(void *service);
 
 struct proxydef childdef = {NULL, 0, 0, S_NOSERVICE, ""};
 
@@ -75,8 +79,12 @@ DWORD WINAPI startsrv(LPVOID data) {
 #else
 void * startsrv(void * data) {
 #endif
-  struct child *d = (struct child *)data;
+  /* counted by start_proxy_thread() before the thread is created */
-  mainfunc(d->argc, (char **)d->argv);
+  mainrun(data);
+  _3proxy_mutex_lock(&runningservices_mutex);
+  runningservices--;
+  _3proxy_mutex_unlock(&runningservices_mutex);
+  _3proxy_sem_unlock(serviceexited);
   return 0;
 }
 
@@ -142,36 +150,45 @@ unsigned char * dologname (unsigned char *buf, unsigned char *name, const unsign
 	return buf;
 }
 
+/* The service is set up here, in the thread reading the configuration, so
+   an error in its options stops the configuration; its own thread then
+   listens and serves. */
 int start_proxy_thread(struct child * chp){
   pthread_t thread;
+  void *service;
+  int res;
 #ifdef _WIN32
   HANDLE h;
 #endif
 
-	_3proxy_sem_lock(conf.threadinit);
+	if((res = mainprepare(chp->argc, (char **)chp->argv, &service))) {
+		fprintf(stderr, "Service not started on line: %d%s\n", linenum, res == 2? ": insufficient memory":"");
+		return(40);
+	}
+	_3proxy_mutex_lock(&runningservices_mutex);
+	runningservices++;
+	_3proxy_mutex_unlock(&runningservices_mutex);
 #ifdef _WIN32
 #ifndef _WINCE
-	h = (HANDLE)_beginthreadex((LPSECURITY_ATTRIBUTES )NULL, 16384+conf.stacksize, (void *)startsrv, (void *) chp, (DWORD)0, &thread);
+	h = (HANDLE)_beginthreadex((LPSECURITY_ATTRIBUTES )NULL, 16384+conf.stacksize, (void *)startsrv, service, (DWORD)0, &thread);
 #else
-	h = (HANDLE)CreateThread((LPSECURITY_ATTRIBUTES )NULL, 16384+conf.stacksize, (void *)startsrv, (void *) chp, (DWORD)0, &thread);
+	h = (HANDLE)CreateThread((LPSECURITY_ATTRIBUTES )NULL, 16384+conf.stacksize, (void *)startsrv, service, (DWORD)0, &thread);
 #endif
 	if(h)CloseHandle(h);
+	else {
 #else
 	pthread_attr_init(&pa);
 	pthread_attr_setstacksize(&pa,threadstacksize(conf.stacksize));
 	pthread_attr_setdetachstate(&pa,PTHREAD_CREATE_DETACHED);
-	if(pthread_create(&thread, &pa, startsrv, (void *)chp)){
+	res = pthread_create(&thread, &pa, startsrv, service);
-		pthread_attr_destroy(&pa);
-		fprintf(stderr, "Failed to create service thread on line %d, try to set larger stacksize\n", linenum);
-		_3proxy_sem_unlock(conf.threadinit);
-		return(40);
-	}
 	pthread_attr_destroy(&pa);
+	if(res){
 #endif
-	_3proxy_sem_lock(conf.threadinit);
+		_3proxy_mutex_lock(&runningservices_mutex);
-	_3proxy_sem_unlock(conf.threadinit);
+		runningservices--;
-	if(haveerror)  {
+		_3proxy_mutex_unlock(&runningservices_mutex);
-		fprintf(stderr, "Service not started on line: %d%s\n", linenum, haveerror == 2? ": insufficient memory":"");
+		maindrop(service);
+		fprintf(stderr, "Failed to create service thread on line %d, try to set larger stacksize\n", linenum);
 		return(40);
 	}
 	return 0;
@@ -187,6 +204,9 @@ static int h_proxy(int argc, unsigned char ** argv){
 		childdef.port = 3128;
 		childdef.isudp = 0;
 		childdef.service = S_PROXY;
+#ifdef WITH_FTP
+		childdef.helpmessage = " -Xftp - fetch ftp:// URLs\n";
+#endif
 #ifdef NOIPV6
 		if(!resolvfunc || (resolvfunc == myresolver && !dns_table.poolsize)){
 			fprintf(stderr, "[line %d] Warning: no nserver/nscache configured, proxy may run very slow\n", linenum);
@@ -358,7 +378,8 @@ static int h_log(int argc, unsigned char ** argv){
 #ifndef _WIN32
 			conf.logfunc = logsyslog;
 			if(notchanged) return 0;
-			openlog((char *)conf.logtarget+1, LOG_PID, LOG_DAEMON);
+			/* connected now: after a chroot there is no /dev/log to find */
+			openlog((char *)conf.logtarget+1, LOG_PID | LOG_NDELAY, LOG_DAEMON);
 #endif
 		}
 #ifdef WITH_ODBC
@@ -1901,6 +1922,14 @@ static int h_setgid(int argc, unsigned char **argv){
 }
 
 
+/* What the C library reads on first use from files a chroot hides, read
+   while they can still be found: the time zone, without which every time
+   is UTC. Names looked up through the system cannot be helped this way, the
+   hosts file and resolv.conf are read again for lookups to come. */
+static void prechroot(void){
+	tzset();
+}
+
 static int h_chroot(int argc, unsigned char **argv){
 	uid_t uid = 0;
 	gid_t gid = 0;
@@ -1920,6 +1949,8 @@ static int h_chroot(int argc, unsigned char **argv){
         }
 	if(!chrootp){
 		char *p;
+
+		prechroot();
 		if(chroot((char *)argv[1])) {
 			fprintf(stderr, "Unable to chroot %s", argv[1]);
 			return(1);
@@ -2263,6 +2294,7 @@ int readconfig(FILE * fp){
 
   if( !(buf = malloc(bufsize)) || ! (argv = malloc((NPARAMS + 1) * sizeof(unsigned char *))) ) {
 		fprintf(stderr, "No memory for configuration");
+		if(buf) free(buf);
 		return(10);
   }
   for (linenum = 1; fgets((char *)buf, STRINGBUF, fp); linenum++){
@@ -2272,7 +2304,8 @@ int readconfig(FILE * fp){
 	argc = parsestr (buf, argv, NPARAMS-1, &buf, &inbuf, &bufsize);
 	if(argc < 1) {
 		fprintf(stderr, "Parse error line %d\n", linenum);
-		return(11);
+		res = 11;
+		goto DONE;
 	}
 	argv[argc] = NULL;
 	if(!strcmp((char *)argv[0], "end") && argc == 1) {	
@@ -2284,23 +2317,28 @@ int readconfig(FILE * fp){
 		if(!strcmp((char *)argv[0], (char *)cm->command)){
 		    if(argc < cm->minargs || (cm->maxargs && argc > cm->maxargs)){
 			fprintf(stderr, "Command: '%s' wrong number of arguments , line %d\n", argv[0], linenum);
-			return(linenum);
+			res = linenum;
+			goto DONE;
 		    }
 		    res = (*cm->handler)(argc, argv);
 		    if(res > 0){
 			fprintf(stderr, "Command: '%s' failed with code %d, line %d\n", argv[0], res, linenum);
-			return(linenum);
+			res = linenum;
+			goto DONE;
 		    }
 		    if(!res) break;
 		}
 	}
 	if(res != 1)continue;
 	fprintf(stderr, "Unknown command: '%s' line %d\n", argv[0], linenum);
-	return(linenum);
+	res = linenum;
+	goto DONE;
   }
+  res = 0;
+DONE:
   free(buf);
   free(argv);
-  return 0;
+  return res;
 
 }
 

src/ftppr.c

@@ -23,14 +23,18 @@ void * ftpprchild(struct clientparam* param) {
  SOCKET sc = INVALID_SOCKET, ss = INVALID_SOCKET, clidatasock = INVALID_SOCKET;
  SASIZETYPE sasize;
  char * req = NULL;
+#ifndef _WIN32
  struct linger lg;
+#endif
  struct pollfd fds;
 
  if(!(buf = malloc(BUFSIZE))) RETURN(876);
  param->ctrlsock = param->clisock;
  param->operation = CONNECT;
+#ifndef _WIN32
  lg.l_onoff = 1;
- lg.l_linger = conf.timeouts[STRING_L];;
+ lg.l_linger = conf.timeouts[STRING_L];
+#endif
  if(socksend(param, param->ctrlsock, (unsigned char *)"220 Ready\r\n", 11, conf.timeouts[STRING_S])!=11) {RETURN (801);}
  for(;;){
 	i = sockgetlinebuf(param, CLIENT, buf, BUFSIZE - 10, '\n', conf.timeouts[CONNECTION_S]);
@@ -262,8 +266,11 @@ void * ftpprchild(struct clientparam* param) {
 		}
 		sc = param->remsock;
 		param->remsock = ss;
+#ifndef _WIN32
+		/* not on Windows, see the accepting side in proxymain.c */
 		param->srv->so._setsockopt(param->sostate, param->remsock, SOL_SOCKET, SO_LINGER, (char *)&lg, sizeof(lg));
 		param->srv->so._setsockopt(param->sostate, clidatasock, SOL_SOCKET, SO_LINGER, (char *)&lg, sizeof(lg));
+#endif
 		param->clisock = clidatasock;
 		res = mapsocket(param, conf.timeouts[CONNECTION_S]);
 		if(param->remsock != INVALID_SOCKET) {

src/httpsrv.c

@@ -459,12 +459,12 @@ static int widepath(const char *path, wchar_t *out, int outchars)
 	char *p;
 
 	if(!strncmp(path, "\\\\?\\", 4)) snprintf(prefixed, sizeof(prefixed), "%s", path);
-	else if(path[0] == '\\\\' && path[1] == '\\\\')
+	else if(path[0] == '\\' && path[1] == '\\')
 		snprintf(prefixed, sizeof(prefixed), "\\\\?\\UNC\\%s", path + 2);
 	else snprintf(prefixed, sizeof(prefixed), "\\\\?\\%s", path);
 
 	/* the extended form takes no forward slashes */
-	for(p = prefixed; *p; p++) if(*p == '/') *p = '\\\\';
+	for(p = prefixed; *p; p++) if(*p == '/') *p = '\\';
 
 	return MultiByteToWideChar(CP_UTF8, 0, prefixed, -1, out, outchars) > 0? 0 : 1;
 }
@@ -481,9 +481,19 @@ static int filemeta(const char *path, struct filemeta *m)
 {
 #ifdef _WIN32
 	wchar_t wide[HTTPSRV_LINE];
+#ifdef __WATCOMC__
+	/* Open Watcom has the 64-bit stat under its older name */
+#ifndef _S_IFREG
+#define _S_IFREG S_IFREG
+#endif
+	struct _stati64 st;
+
+	if(widepath(path, wide, HTTPSRV_LINE) || _wstati64(wide, &st)) return 1;
+#else
 	struct _stat64 st;
 
 	if(widepath(path, wide, HTTPSRV_LINE) || _wstat64(wide, &st)) return 1;
+#endif
 	m->size = (uint64_t)st.st_size;
 	m->mtime = st.st_mtime;
 	m->isreg = (st.st_mode & _S_IFREG) != 0;
@@ -1114,6 +1124,10 @@ void freehttprules(struct httprule *rule)
 		next = rule->next;
 		if(rule->host.name) free(rule->host.name);
 		if(rule->url.name) free(rule->url.name);
+#ifdef WITH_PCRE
+		if(rule->host.re) pcre_pattern_free(rule->host.re);
+		if(rule->url.re) pcre_pattern_free(rule->url.re);
+#endif
 		if(rule->params) free(rule->params);
 		if(rule->ctype) free(rule->ctype);
 		if(rule->hdrs) free(rule->hdrs);
@@ -1154,6 +1168,37 @@ static void httpsrv_drain(struct clientparam *param, uint64_t len)
 	}
 }
 
+/* Ends a connection this server is closing while the client may still be
+   sending: a body which was not read, or a request sent behind one answered
+   with a close. Closing with that unread resets the connection, and the
+   client loses the answer. The sending side is shut down instead and what
+   arrives is thrown away until the client closes too, for a bounded time
+   and amount, which is what lets the answer be read. */
+static void httpsrv_linger(struct clientparam *param)
+{
+	char buf[HTTPSRV_BLOCK];
+	struct pollfd fd;
+	uint64_t left = HTTPSRV_MAXBODY;
+	time_t deadline = time(NULL) + conf.timeouts[LINGER_TO];
+
+	if(param->clisock == INVALID_SOCKET) return;
+	if(param->srv->so._shutdown(param->sostate, param->clisock, SHUT_WR)) return;
+	while(left){
+		int timeo = (int)(deadline - time(NULL));
+		int got;
+
+		if(timeo <= 0) break;
+		memset(&fd, 0, sizeof(fd));
+		fd.fd = param->clisock;
+		fd.events = POLLIN;
+		if(param->srv->so._poll(param->sostate, &fd, 1, timeo * 1000) <= 0) break;
+		got = (int)param->srv->so._recv(param->sostate, param->clisock, buf,
+			(left > sizeof(buf))? (int)sizeof(buf) : (int)left, 0);
+		if(got <= 0) break;
+		left -= (uint64_t)got;
+	}
+}
+
 /* Keeps the request as the client wrote it. Only what a handoff needs: the
    request line and the headers, exactly as they came, since the proxy code
    reads them again from the beginning. */
@@ -1451,6 +1496,12 @@ static int httpsrv_request(struct clientparam *param, struct httpreq *r)
 		}
 	}
 	if(i && i != 10){
+		/* The body goes before the answer, as for any answer this server
+		   gives: left in the stream it would be read as the next request. */
+		if(!r->drained){
+			if(r->contentlen) httpsrv_drain(param, r->contentlen);
+			r->drained = 1;
+		}
 		/* 4 no credentials, 5 unknown user, 6 wrong password: all of them
 		   should let the client offer credentials again. */
 		if(i >= 4 && i <= 6) op_authrequired(r);
@@ -1573,7 +1624,12 @@ void * httpsrvchild(struct clientparam *param)
 			dolog(param, (unsigned char *)logbuf);
 		}
 
-		if(handoff || !r.keepalive) break;
+		if(handoff) break;
+		if(!r.keepalive){
+			/* the client may still be sending; let it read the answer */
+			httpsrv_linger(param);
+			break;
+		}
 		first = 0;
 	}
 	/* A child named by an access rule takes the connection over, which the

src/proxy.c

@@ -171,6 +171,20 @@ static int growbuf(unsigned char **buf, int *bufsize, int need){
 	return 0;
 }
 
+/* Whether a request line names the host the previous one did, whatever its
+   method: the connection to the server is kept for the same host only, and
+   a POST after a GET to it is no reason to open another one. */
+static int samehost(const unsigned char *line, int len, const unsigned char *host, int hostlen){
+	const unsigned char *p = (const unsigned char *)memchr(line, ' ', len);
+
+	if(!p || hostlen <= 0) return 0;
+	p++;
+	if(!strncasecmp((const char *)p, "http://", 7)) p += 7;
+	if((int)(p - line) + hostlen >= len || strncasecmp((const char *)p, (const char *)host, hostlen))
+		return 0;
+	return p[hostlen] == '/' || p[hostlen] == ' ';
+}
+
 static void freeptr(void *p){
 	void **pp = (void **)p;
 	if(*pp) { free(*pp); *pp = NULL; }
@@ -285,7 +299,7 @@ void * proxychild(struct clientparam* param) {
  int clhdrofs_req = -1;
  int isconnect = 0;
  int redirect = 0;
- int prefix = 0, ckeepalive=0;
+ int prefix = 0, hostofs = 0, ckeepalive=0;
  int ftp = 0;
  int ssoff = 0;
 #ifndef ANONYMOUS
@@ -354,7 +368,7 @@ for(;;){
  buf[i] = 0;
  
  if(req) {
-	if(!param->transparent && !param->srv->transparent && (i<=prefix || strncasecmp((char *)buf, (char *)req, prefix))){
+	if(!param->transparent && !param->srv->transparent && !samehost(buf, i, req + hostofs, prefix - hostofs)){
 		ckeepalive = 0;
 		if(param->remsock != INVALID_SOCKET){
 			param->srv->so._shutdown(param->sostate, param->remsock, SHUT_RDWR);
@@ -387,6 +401,7 @@ for(;;){
 	}
 #ifdef WITH_FTP
 	else if (!strncasecmp((char *)sb, "ftp://", 6)) {
+		if(!param->srv->allowftp) RETURN(513);
 		ftp = 1;
 		sb += 6;
 	}
@@ -411,6 +426,7 @@ for(;;){
  		*se = 0;
 	}
 	prefix = (int)(se - buf);
+	hostofs = (int)(sb - buf);
 	su = (unsigned char*)strrchr((char *)sb, '@');
 	if(su) {
 		su = (unsigned char *)strdup((char *)sb);
@@ -1357,6 +1373,9 @@ struct proxydef childdef = {
 	S_PROXY,
 	"-a - anonymous proxy\r\n"
 	"-a1 - anonymous proxy with random client IP spoofing\r\n"
+#ifdef WITH_FTP
+	"-Xftp - fetch ftp:// URLs\r\n"
+#endif
 };
 #include "proxymain.c"
 #endif

src/proxy.h

@@ -239,6 +239,11 @@ extern struct extparam conf;
 
 extern int timeouts[12];
 
+/* directions sockmap() passes with splice(): -s1, -s2, -s3; with
+   SPLICE_AUTO (-s alone, -s5 to -s7) only when it pays */
+#define SPLICE_DOWNLOAD 1
+#define SPLICE_UPLOAD 2
+#define SPLICE_AUTO 4
 int sockmap(struct clientparam * param, int timeo, int usesplice);
 int udpsockmap(struct clientparam * param, int timeo);
 int udpbind(struct clientparam * param);
@@ -322,13 +327,23 @@ extern _3proxy_sem_t udpinit;
 #define _3proxy_sem_init(x, count, maxcount) (((x) = CreateSemaphore(NULL, (count), (maxcount), NULL))? 0 : 1)
 #define _3proxy_sem_lock(x) WaitForSingleObject(x, INFINITE)
 #define _3proxy_sem_unlock(x) ReleaseSemaphore(x, 1, NULL)
+#define _3proxy_sem_init_timed(x, count, maxcount) _3proxy_sem_init(x, (count), (maxcount))
+#define _3proxy_sem_timedlock(x, ms) (WaitForSingleObject(x, ms) != WAIT_OBJECT_0)
+#define _3proxy_sem_destroy(x) CloseHandle(x)
 #else
-int _3proxy_sem_init_f(_3proxy_sem_t *sem, unsigned count, unsigned maxcount);
+int _3proxy_sem_init_f(_3proxy_sem_t *sem, unsigned count, unsigned maxcount, int timed);
 void _3proxy_sem_lock_f(_3proxy_sem_t *sem);
+int _3proxy_sem_timedlock_f(_3proxy_sem_t *sem, unsigned ms);
 void _3proxy_sem_unlock_f(_3proxy_sem_t *sem);
-#define _3proxy_sem_init(x, count, maxcount) _3proxy_sem_init_f(&x, (count), (maxcount))
+void _3proxy_sem_destroy_f(_3proxy_sem_t *sem);
+#define _3proxy_sem_init(x, count, maxcount) _3proxy_sem_init_f(&x, (count), (maxcount), 0)
+/* for a semaphore waited on with a timeout: the wait is not stretched or cut
+   short by the system time being changed, where the platform allows it */
+#define _3proxy_sem_init_timed(x, count, maxcount) _3proxy_sem_init_f(&x, (count), (maxcount), 1)
 #define _3proxy_sem_lock(x) _3proxy_sem_lock_f(&x)
+#define _3proxy_sem_timedlock(x, ms) _3proxy_sem_timedlock_f(&x, (ms))
 #define _3proxy_sem_unlock(x) _3proxy_sem_unlock_f(&x)
+#define _3proxy_sem_destroy(x) _3proxy_sem_destroy_f(&x)
 #endif
 
 
@@ -457,6 +472,10 @@ struct dictionary;
 struct node;
 struct property;
 extern _3proxy_mutex_t config_mutex;
+/* services running, and a post each time one of them has finished */
+extern int runningservices;
+extern _3proxy_mutex_t runningservices_mutex;
+extern _3proxy_sem_t serviceexited;
 extern _3proxy_mutex_t bandlim_mutex;
 extern _3proxy_mutex_t connlim_mutex;
 extern _3proxy_mutex_t tc_mutex;
@@ -467,7 +486,7 @@ extern struct datatype datatypes[64];
 extern struct commands commandhandlers[];
 
 #ifdef WITHSPLICE
-#define mapsocket(a,b) ((a->srv->usesplice && !a->ndatfilterssrv && !a->ndatfilterscli)?sockmap(a,b,1):sockmap(a,b,0))
+#define mapsocket(a,b) ((a->srv->usesplice && !a->ndatfilterssrv && !a->ndatfilterscli)?sockmap(a,b,a->srv->usesplice):sockmap(a,b,0))
 #else
 #define mapsocket(a,b) sockmap(a,b, 0)
 #endif
@@ -487,7 +506,8 @@ extern struct socketoptions {
 	int opt;
 	char * optname;
 } sockopts[];
-void setopts(SOCKET s, int opts);
+int getopts(const char *s, struct sockoptset *opts);
+void setopts(SOCKET s, const struct sockoptset *opts);
 char * printopts(char *sep);
 
 #ifdef _WINCE

src/sem.c

@@ -0,0 +1,110 @@
+/*
+   3APA3A simplest proxy server
+   (c) 2002-2026 by Vladimir Dubrovin <vlad@3proxy.org>
+
+   please read License Agreement
+
+*/
+
+/* Counting semaphores for POSIX systems, built on a mutex and a condition
+   variable: unnamed POSIX semaphores are not available everywhere (macOS)
+   and have no timed wait there. Windows uses its own semaphores, see proxy.h. */
+
+#include "proxy.h"
+
+#ifndef _WIN32
+/* macOS has CLOCK_MONOTONIC but no pthread_condattr_setclock(); a relative
+   wait is used there instead, which the system time does not affect either */
+#if defined(CLOCK_MONOTONIC) && !defined(__APPLE__)
+#define SEM_MONOTONIC
+#endif
+
+int _3proxy_sem_init_f(_3proxy_sem_t *sem, unsigned count, unsigned maxcount, int timed){
+	int res;
+
+	sem->count = count;
+	sem->maxcount = maxcount;
+	sem->monotonic = 0;
+	if(pthread_mutex_init(&sem->mutex, NULL)) return 1;
+#ifdef SEM_MONOTONIC
+	if(timed){
+		pthread_condattr_t attr;
+
+		if(!pthread_condattr_init(&attr)){
+			if(!pthread_condattr_setclock(&attr, CLOCK_MONOTONIC)) sem->monotonic = 1;
+			res = pthread_cond_init(&sem->cond, &attr);
+			pthread_condattr_destroy(&attr);
+		}
+		else res = pthread_cond_init(&sem->cond, NULL);
+	}
+	else
+#endif
+	res = pthread_cond_init(&sem->cond, NULL);
+	if(res){
+		pthread_mutex_destroy(&sem->mutex);
+		return 1;
+	}
+	return 0;
+}
+
+void _3proxy_sem_lock_f(_3proxy_sem_t *sem){
+	pthread_mutex_lock(&sem->mutex);
+	while(!sem->count) pthread_cond_wait(&sem->cond, &sem->mutex);
+	sem->count--;
+	pthread_mutex_unlock(&sem->mutex);
+}
+
+/* Returns 0 with the semaphore taken, nonzero if ms passed first. */
+int _3proxy_sem_timedlock_f(_3proxy_sem_t *sem, unsigned ms){
+	struct timespec ts;
+	int res = 0;
+
+#ifdef __APPLE__
+	ts.tv_sec = ms / 1000;
+	ts.tv_nsec = (long)(ms % 1000) * 1000000;
+	pthread_mutex_lock(&sem->mutex);
+	while(!sem->count && !res) res = pthread_cond_timedwait_relative_np(&sem->cond, &sem->mutex, &ts);
+#else
+	/* the deadline is on the clock the condition variable was set up with */
+#ifdef SEM_MONOTONIC
+	if(sem->monotonic) clock_gettime(CLOCK_MONOTONIC, &ts);
+	else
+#endif
+	{
+		struct timeval tv;
+
+		gettimeofday(&tv, NULL);
+		ts.tv_sec = tv.tv_sec;
+		ts.tv_nsec = tv.tv_usec * 1000;
+	}
+	ts.tv_sec += ms / 1000;
+	ts.tv_nsec += (long)(ms % 1000) * 1000000;
+	if(ts.tv_nsec >= 1000000000){
+		ts.tv_sec++;
+		ts.tv_nsec -= 1000000000;
+	}
+	pthread_mutex_lock(&sem->mutex);
+	while(!sem->count && !res) res = pthread_cond_timedwait(&sem->cond, &sem->mutex, &ts);
+#endif
+	if(sem->count){
+		sem->count--;
+		res = 0;
+	}
+	pthread_mutex_unlock(&sem->mutex);
+	return res;
+}
+
+void _3proxy_sem_unlock_f(_3proxy_sem_t *sem){
+	pthread_mutex_lock(&sem->mutex);
+	if(sem->count < sem->maxcount){
+		sem->count++;
+		pthread_cond_signal(&sem->cond);
+	}
+	pthread_mutex_unlock(&sem->mutex);
+}
+
+void _3proxy_sem_destroy_f(_3proxy_sem_t *sem){
+	pthread_cond_destroy(&sem->cond);
+	pthread_mutex_destroy(&sem->mutex);
+}
+#endif

src/sockmap.c

@@ -39,6 +39,18 @@ ssize_t splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t le
 
 #define MAXSPLICE 65536
 
+/* With SPLICE_AUTO, when a direction allowed to splice actually does
+   (without it, an allowed direction always does).
+   A body of known length is spliced if at least SPLICE_MIN of it is left
+   to read once what is already buffered is counted: a smaller one, or one
+   read whole with its headers, is cheaper to copy.
+   A stream of unknown length starts copied and is spliced after
+   SPLICE_STREAK reads in a row of at least SPLICE_MIN each, and goes back to
+   copying after SPLICE_STREAK+1 splices in a row of less than SPLICE_SMALL. */
+#define SPLICE_MIN 16384
+#define SPLICE_SMALL 4096
+#define SPLICE_STREAK 2
+
 #endif
 
 #define MIN(a,b) ((a>b)?b:a)
@@ -49,6 +61,31 @@ ssize_t splice(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t le
    before the sockets are closed. */
 #define SESSIONEND (!halfclose && (CLIENTTERMREAD || SERVERTERMREAD))
 
+/* A hang-up on a socket there is data to write to. With poll() it comes
+   once both sides are shut down, and the write tried then fails, which ends
+   it. WSAPoll reports it as soon as the peer has half-closed, a peer still
+   reading, and keeps reporting it whether the socket has room or not: room
+   is waited for with select() then, up to timeo seconds. Returns whether to
+   try the write. */
+static int hupwrite(SOCKET s, short revents, int timeo){
+#ifdef _WIN32
+	if(!(revents & POLLOUT)){
+		fd_set wfd;
+		struct timeval tv;
+
+		FD_ZERO(&wfd);
+		FD_SET(s, &wfd);
+		tv.tv_sec = timeo;
+		tv.tv_usec = 0;
+		if(select(0, NULL, &wfd, NULL, &tv) < 1) return 0;
+	}
+#endif
+	return 1;
+}
+
+/* usesplice is a mask of the directions passed with splice() rather than
+   through a buffer: SPLICE_DOWNLOAD, server to client, and SPLICE_UPLOAD,
+   client to server. */
 int sockmap(struct clientparam * param, int timeo, int usesplice){
  int halfclose = param->srv->halfclose;
  uint64_t fromclient=0x7fffffffffffffff, fromserver =0x7fffffffffffffff;
@@ -60,6 +97,10 @@ int sockmap(struct clientparam * param, int timeo, int usesplice){
 #endif
  int HASERROR=0;
  int CLIENTTERMREAD = 0, CLIENTTERMWRITE = 0, SERVERTERMREAD = 0, SERVERTERMWRITE = 0;
+ /* the side has hung up: closed by the peer after its own was shut down */
+ int CLIENTHUP = 0, SERVERHUP = 0;
+ /* whether the side's socket is in this round's poll() */
+ int clipolled = 1, srvpolled = 1;
  int after = 0;
  struct pollfd fds[8];
  int fdsc = 0;
@@ -76,16 +117,30 @@ int sockmap(struct clientparam * param, int timeo, int usesplice){
 #ifdef WITHSPLICE
  uint64_t inclientpipe = 0, inserverpipe = 0;
  int TOCLIENTPIPE = 0, FROMCLIENTPIPE = 0, TOSERVERPIPE = 0, FROMSERVERPIPE = 0;
- int pipesrv[2] = {-1,-1};
+ /* The client's data goes through pipecli, the server's through pipesrv.
- int pipecli[2] = {-1,-1};
+    They belong to the connection: made when first needed, kept for the
+    next mapping, closed with the connection. */
+ int *pipecli = param->pipecli, *pipesrv = param->pipesrv;
+ /* allowed to splice (-s), splicing now, and a run of reads counted
+    towards switching; wantbuf: go back to copying once the pipe is empty */
+ int allowcli = 0, allowsrv = 0, splicecli = 0, splicesrv = 0, autosplice;
+ int streakcli = 0, streaksrv = 0, wantbufcli = 0, wantbufsrv = 0;
 
  if((!param->nolongdatfilter && (param->ndatfilterscli > 0 || param->ndatfilterssrv))) usesplice = 0;
- if(usesplice){
+ allowcli = (usesplice & SPLICE_UPLOAD) != 0;
-	TOCLIENTPIPE = FROMCLIENTPIPE = TOSERVERPIPE = FROMSERVERPIPE = 1;
+ allowsrv = (usesplice & SPLICE_DOWNLOAD) != 0;
-	TOCLIENTBUF = TOSERVERBUF = 0;
+ autosplice = (usesplice & SPLICE_AUTO) != 0;
-	if(pipe2(pipecli, O_NONBLOCK) < 0) RETURN (21);
+ if(!autosplice){
-	if(pipe2(pipesrv, O_NONBLOCK) < 0) RETURN (21);
+	splicecli = allowcli;
+	splicesrv = allowsrv;
  }
+ TOCLIENTPIPE = FROMCLIENTPIPE = allowcli;
+ TOSERVERPIPE = FROMSERVERPIPE = allowsrv;
+#else
+#define splicecli 0
+#define splicesrv 0
+#define wantbufcli 0
+#define wantbufsrv 0
 #endif
 
  inserverbuf = param->srvinbuf - param->srvoffset;
@@ -108,17 +163,17 @@ int sockmap(struct clientparam * param, int timeo, int usesplice){
  if(inserverbuf >= fromserver) FROMSERVER = 0;
  if(inclientbuf >= fromclient) FROMCLIENT = 0;
 #ifdef WITHSPLICE
- if(!usesplice)
+ /* a body of known length large enough is spliced from the start */
+ if(autosplice && allowsrv && param->waitserver64 && fromserver > inserverbuf && fromserver - inserverbuf >= SPLICE_MIN) splicesrv = 1;
+ if(autosplice && allowcli && param->waitclient64 && fromclient > inclientbuf && fromclient - inclientbuf >= SPLICE_MIN) splicecli = 1;
 #endif
- {
+ /* a spliced direction needs no buffer, only what was read before */
-	if(fromserver && !param->srvbuf && (!(param->srvbuf=malloc(SRVBUFSIZE)) || !(param->srvbufsize = SRVBUFSIZE))){
+ if(!splicesrv && fromserver && !param->srvbuf && (!(param->srvbuf=malloc(SRVBUFSIZE)) || !(param->srvbufsize = SRVBUFSIZE))){
 	RETURN (21);
  }
-	if(fromclient && !param->clibuf && (!(param->clibuf=malloc(SRVBUFSIZE)) || !(param->clibufsize = SRVBUFSIZE))){
+ if(!splicecli && fromclient && !param->clibuf && (!(param->clibuf=malloc(SRVBUFSIZE)) || !(param->clibufsize = SRVBUFSIZE))){
 	RETURN (21);
  }
-
- }
  if(param->srvinbuf == param->srvoffset) param->srvinbuf =param->srvoffset = 0;
  if(param->cliinbuf == param->clioffset) param->cliinbuf =param->clioffset = 0;
  if(param->clibufsize == param->cliinbuf) TOCLIENTBUF = 0;
@@ -292,8 +347,21 @@ log("done send to client from buf");
 		}
 	}
 #ifdef WITHSPLICE
-	if(usesplice){
+	/* a direction going back to copying does once its pipe is empty */
-		if(inclientpipe && !inclientbuf && FROMCLIENTPIPE && TOSERVER){
+	if(wantbufcli && !inclientpipe){
+		splicecli = wantbufcli = streakcli = 0;
+		if(!param->clibuf && (!(param->clibuf=malloc(SRVBUFSIZE)) || !(param->clibufsize = SRVBUFSIZE))) RETURN (21);
+		if(param->cliinbuf < param->clibufsize) TOCLIENTBUF = 1;
+	}
+	if(wantbufsrv && !inserverpipe){
+		splicesrv = wantbufsrv = streaksrv = 0;
+		if(!param->srvbuf && (!(param->srvbuf=malloc(SRVBUFSIZE)) || !(param->srvbufsize = SRVBUFSIZE))) RETURN (21);
+		if(param->srvinbuf < param->srvbufsize) TOSERVERBUF = 1;
+	}
+	{
+		/* each direction goes through its pipe if spliced, through its
+		   buffer otherwise; what was in the buffer before is sent first */
+		if(splicecli && inclientpipe && !inclientbuf && FROMCLIENTPIPE && TOSERVER){
 #ifdef WITHLOG
 log("send to server from pipe");
 #endif
@@ -330,7 +398,7 @@ log(logbuf);
 				}
 			}
 		}
-		if(inserverpipe && !inserverbuf && FROMSERVERPIPE && TOCLIENT){
+		if(splicesrv && inserverpipe && !inserverbuf && FROMSERVERPIPE && TOCLIENT){
 #ifdef WITHLOG
 log("send to client from pipe");
 #endif
@@ -360,12 +428,17 @@ log(logbuf);
 				}
 			}
 		}
-		if(fromclient>inclientpipe && FROMCLIENT && TOCLIENTPIPE && !SESSIONEND){
+		if(splicecli && !wantbufcli && fromclient>inclientpipe && FROMCLIENT && TOCLIENTPIPE && !SESSIONEND){
-			int error;
-			socklen_t len=sizeof(error);
 #ifdef WITHLOG
 log("read from client to pipe");
 #endif
+			if(pipecli[0] < 0 && pipe2(pipecli, O_NONBLOCK) < 0){
+				/* no pipe to be had: copy instead */
+				allowcli = splicecli = TOCLIENTPIPE = FROMCLIENTPIPE = 0;
+				if(!param->clibuf && (!(param->clibuf=malloc(SRVBUFSIZE)) || !(param->clibufsize = SRVBUFSIZE))) RETURN (21);
+				if(param->cliinbuf < param->clibufsize) TOCLIENTBUF = 1;
+				continue;
+			}
 			errno = 0;
 			res = splice(param->clisock, NULL, pipecli[1], NULL, (int)MIN((uint64_t)MAXSPLICE - inclientpipe, (uint64_t)fromclient-inclientpipe), SPLICE_F_NONBLOCK|SPLICE_F_MOVE);
 #ifdef WITHLOG
@@ -396,19 +469,27 @@ log("done read from client to pipe");
 				graceclinum++;
 				graceclitraf += res;
 				gracesrvnum = gracesrvtraf = 0;
+				if(autosplice && !param->waitclient64){
+					if(res >= SPLICE_SMALL) streakcli = 0;
+					else if(++streakcli > SPLICE_STREAK) wantbufcli = 1;
+				}
 				inclientpipe += res;
 				if(inclientpipe >= MAXSPLICE) TOCLIENTPIPE = 0;
 				needaction = 0;
 				continue;
 			}
 		}
-		if(fromserver > inserverpipe && FROMSERVER && TOSERVERPIPE && !SESSIONEND){
+		if(splicesrv && !wantbufsrv && fromserver > inserverpipe && FROMSERVER && TOSERVERPIPE && !SESSIONEND){
-			int error; 
-			socklen_t len=sizeof(error);
-			errno = 0;
 #ifdef WITHLOG
 log("read from server to pipe\n");
 #endif
+			if(pipesrv[0] < 0 && pipe2(pipesrv, O_NONBLOCK) < 0){
+				allowsrv = splicesrv = TOSERVERPIPE = FROMSERVERPIPE = 0;
+				if(!param->srvbuf && (!(param->srvbuf=malloc(SRVBUFSIZE)) || !(param->srvbufsize = SRVBUFSIZE))) RETURN (21);
+				if(param->srvinbuf < param->srvbufsize) TOSERVERBUF = 1;
+				continue;
+			}
+			errno = 0;
 			res = splice(param->remsock, NULL, pipesrv[1], NULL, MIN(MAXSPLICE - inserverpipe, fromserver - inserverpipe), SPLICE_F_NONBLOCK|SPLICE_F_MOVE);
 #ifdef WITHLOG
 log("server to pipe splice finished\n");
@@ -438,6 +519,10 @@ log("done read from server to pipe\n");
 				gracesrvnum++;
 				gracesrvtraf += res;
 				graceclinum = graceclitraf = 0;
+				if(autosplice && !param->waitserver64){
+					if(res >= SPLICE_SMALL) streaksrv = 0;
+					else if(++streaksrv > SPLICE_STREAK) wantbufsrv = 1;
+				}
 			    	param->nreads++;
 				param->statssrv64 += res;
 				inserverpipe += res;
@@ -452,10 +537,9 @@ log("done read from server to pipe\n");
 			}
 		}
 	}
-	else
 #endif
 	{
-		if(fromclient > inclientbuf && FROMCLIENT && TOCLIENTBUF && !SESSIONEND){
+		if(!splicecli && fromclient > inclientbuf && FROMCLIENT && TOCLIENTBUF && !SESSIONEND){
 #ifdef WITHLOG
 log("read from client to buf");
 #endif
@@ -480,6 +564,17 @@ log("done read from client to buf");
 				graceclinum++;
 				graceclitraf += res;
 				gracesrvnum = gracesrvtraf = 0;
+#ifdef WITHSPLICE
+				if(autosplice && allowcli && !param->waitclient64){
+					if(res < SPLICE_MIN) streakcli = 0;
+					else if(++streakcli >= SPLICE_STREAK){
+						/* a stream: splice from now on, after what is
+						   buffered has been sent */
+						splicecli = TOCLIENTPIPE = FROMCLIENTPIPE = 1;
+						streakcli = 0;
+					}
+				}
+#endif
 				inclientbuf += res;
 				param->cliinbuf += res;
 				if(param->clibufsize == param->cliinbuf) TOCLIENTBUF = 0;
@@ -488,7 +583,7 @@ log("done read from client to buf");
 			}
 		}
 
-		if(fromserver > inserverbuf && FROMSERVER && TOSERVERBUF && !SESSIONEND){
+		if(!splicesrv && fromserver > inserverbuf && FROMSERVER && TOSERVERBUF && !SESSIONEND){
 #ifdef WITHLOG
 log("read from server to buf");
 #endif
@@ -513,6 +608,15 @@ log("done read from server to buf");
 				gracesrvnum++;
 				gracesrvtraf += res;
 				graceclinum = graceclitraf = 0;
+#ifdef WITHSPLICE
+				if(autosplice && allowsrv && !param->waitserver64){
+					if(res < SPLICE_MIN) streaksrv = 0;
+					else if(++streaksrv >= SPLICE_STREAK){
+						splicesrv = TOSERVERPIPE = FROMSERVERPIPE = 1;
+						streaksrv = 0;
+					}
+				}
+#endif
 			    	param->nreads++;
 				param->statssrv64 += res;
 				inserverbuf += res;
@@ -553,15 +657,8 @@ log("done read from server to buf");
 //		if(!CLIENTTERMREAD || !CLIENTTERMWRITE){
 			if(!after){
 				fds[fdsc].fd = param->clisock;
-				if(fromclient && !CLIENTTERMREAD && !FROMCLIENT && !SESSIONEND && ((
+				if(fromclient && !CLIENTTERMREAD && !FROMCLIENT && !SESSIONEND &&
-#ifdef WITHSPLICE
+					((!splicecli && TOCLIENTBUF) || (splicecli && !wantbufcli)))
-					!usesplice &&
-#endif
-					TOCLIENTBUF)
-#ifdef WITHSPLICE
-					|| (usesplice)
-#endif
-						))
 					cli_events |= POLLIN;
 				if(!TOCLIENT && !CLIENTTERMWRITE && (inserverbuf
 #ifdef WITHSPLICE
@@ -569,6 +666,11 @@ log("done read from server to buf");
 #endif
 						))
 					cli_events |= POLLOUT;
+				/* poll() reports a hang-up whatever is asked, and would
+				   return at once every time: the socket is left out then,
+				   rather than given as an invalid one, which WSAPoll takes
+				   for an error */
+				clipolled = cli_events || !CLIENTHUP;
 				if(cli_events){
 					fds[fdsc].fd = param->clisock;
 					fds[fdsc].events = cli_events;
@@ -580,6 +682,8 @@ log("wait writing to client");
 #endif
 				}
 			}
+			else if(!clipolled){
+			}
 			else if(cli_events){
 				if(fds[fdsc].revents &  (POLLERR|POLLNVAL)) {
 					CLIENTTERMREAD = 1;
@@ -601,27 +705,27 @@ log("ready to write to client");
 					}
 					if(fds[fdsc].revents &  (POLLHUP)) {
 						if(fds[fdsc].events & POLLIN) FROMCLIENT = 1;
-						if(fds[fdsc].events & POLLOUT) CLIENTTERMWRITE = 1;
+						if(fds[fdsc].events & POLLOUT){
+							if(hupwrite(param->clisock, fds[fdsc].revents, timeo)) TOCLIENT = 1;
+							else CLIENTTERMWRITE = 1;
 						}
 					}
-			} else if(fds[fdsc].revents & (POLLERR|POLLNVAL|POLLHUP)) {
+				}
+			} else if(fds[fdsc].revents & (POLLERR|POLLNVAL)) {
 				CLIENTTERMREAD = 1;
 				CLIENTTERMWRITE = 1;
+			} else if(fds[fdsc].revents & POLLHUP) {
+				/* what the client sent before it closed is still to be
+				   read, once the buffer has room for it */
+				CLIENTHUP = 1;
 			}
-			fdsc++;
+			if(clipolled) fdsc++;
 
 //		if(!SERVERTERMREAD || !SERVERTERMWRITE){
 			if(!after){
 				fds[fdsc].fd = param->remsock;
-				if(fromserver && !SERVERTERMREAD && !FROMSERVER && !SESSIONEND && ((
+				if(fromserver && !SERVERTERMREAD && !FROMSERVER && !SESSIONEND &&
-#ifdef WITHSPLICE
+					((!splicesrv && TOSERVERBUF) || (splicesrv && !wantbufsrv)))
-					!usesplice &&
-#endif
-					TOSERVERBUF)
-#ifdef WITHSPLICE
-					|| (usesplice)
-#endif
-						))
 					srv_events |= POLLIN;
 				if(!TOSERVER && !SERVERTERMWRITE && (inclientbuf
 #ifdef WITHSPLICE
@@ -629,6 +733,7 @@ log("ready to write to client");
 #endif
 						))
 					srv_events |= POLLOUT;
+				srvpolled = srv_events || !SERVERHUP;
 				if(srv_events){
 					fds[fdsc].fd = param->remsock;
 					fds[fdsc].events = srv_events;
@@ -640,6 +745,8 @@ log("wait writing from server");
 #endif
 				}
 			}
+			else if(!srvpolled){
+			}
 			else if(srv_events){
 				if(fds[fdsc].revents &  (POLLERR|POLLNVAL)) {
 #ifdef WITHLOG
@@ -668,18 +775,24 @@ log("ready to write to server");
 log("server terminated connection");
 #endif
 						if(fds[fdsc].events & POLLIN) FROMSERVER = 1;
-						if(fds[fdsc].events & POLLOUT) SERVERTERMWRITE = 1;
+						if(fds[fdsc].events & POLLOUT){
+							if(hupwrite(param->remsock, fds[fdsc].revents, timeo)) TOSERVER = 1;
+							else SERVERTERMWRITE = 1;
 						}
 					}
-			} else if(fds[fdsc].revents & (POLLERR|POLLNVAL|POLLHUP)) {
+				}
+			} else if(fds[fdsc].revents & (POLLERR|POLLNVAL)) {
 				SERVERTERMREAD = 1;
 				SERVERTERMWRITE = 1;
+			} else if(fds[fdsc].revents & POLLHUP) {
+				/* the rest of the answer may still be waiting to be read */
+				SERVERHUP = 1;
 			}
-			fdsc++;
+			if(srvpolled) fdsc++;
 //		}
 #ifdef WITHSPLICE
-		if(usesplice){
+		{
-			if(fromclient>inclientpipe && !TOCLIENTPIPE && inclientpipe < MAXSPLICE){
+			if(splicecli && fromclient>inclientpipe && !TOCLIENTPIPE && inclientpipe < MAXSPLICE){
 				if(!after){
 #ifdef WITHLOG
 log("wait writing to client pipe");
@@ -700,7 +813,7 @@ log("ready to write to client pipe");
 				}
 				fdsc++;
 			}
-			if(inclientpipe && !FROMCLIENTPIPE){
+			if(splicecli && inclientpipe && !FROMCLIENTPIPE){
 				if(!after){
 #ifdef WITHLOG
 log("wait reading from client pipe");
@@ -719,7 +832,7 @@ log("ready reading from client pipe");
 				}
 				fdsc++;
 			}
-			if(fromserver>inserverpipe && !TOSERVERPIPE && inserverpipe < MAXSPLICE){
+			if(splicesrv && fromserver>inserverpipe && !TOSERVERPIPE && inserverpipe < MAXSPLICE){
 				if(!after){
 #ifdef WITHLOG
 log("wait writing to server pipe");
@@ -738,7 +851,7 @@ log("ready writing to server pipe");
 				}
 				fdsc++;
 			}
-			if(inserverpipe && !FROMSERVERPIPE){
+			if(splicesrv && inserverpipe && !FROMSERVERPIPE){
 				if(!after){
 #ifdef WITHLOG
 log("wait reading from server pipe");
@@ -819,18 +932,29 @@ log("timeout");
  else if(inclientpipe || inserverpipe) res = 94;
 #endif
 
- if((param->nwrites > 0 && !SERVERTERMWRITE) || (param->nreads > 0 && !CLIENTTERMWRITE))
-	usleep(SLEEPTIME * 10);
-
 CLEANRET:
 
 #ifdef WITHSPLICE
- if(pipecli[0] >= 0) close(pipecli[0]);
+ /* empty pipes stay with the connection for its next mapping; one still
- if(pipecli[1] >= 0) close(pipecli[1]);
+    holding data, left by an error or a timeout, cannot be used again */
- if(pipesrv[0] >= 0) close(pipesrv[0]);
+ if(inclientpipe && pipecli[0] >= 0){
- if(pipesrv[1] >= 0) close(pipesrv[1]);
+	close(pipecli[0]);
+	close(pipecli[1]);
+	pipecli[0] = pipecli[1] = -1;
+ }
+ if(inserverpipe && pipesrv[0] >= 0){
+	close(pipesrv[0]);
+	close(pipesrv[1]);
+	pipesrv[0] = pipesrv[1] = -1;
+ }
 #endif
 
  return res;
 }
 #undef SESSIONEND
+#ifndef WITHSPLICE
+#undef splicecli
+#undef splicesrv
+#undef wantbufcli
+#undef wantbufsrv
+#endif

src/socks.c

@@ -184,7 +184,7 @@ void * sockschild(struct clientparam* param) {
 	 param->operation = command == 2?BIND:UDPASSOC;
 	 if(command == 2){
 		if ((param->remsock=param->srv->so._socket(param->sostate, SASOCK(&param->req), SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) {RETURN (11);}
-		setopts(param->remsock, param->srv->lissockopts);
+		setopts(param->remsock, &param->srv->lissockopts);
 		if(extbindtodevice(param, param->remsock)) {RETURN (12);}
 	 }
 	 break;
@@ -272,13 +272,18 @@ CLEANRET:
 	int repcode;
 
 	sasize = sizeof(sin);
-	if(command != 3 && param->remsock != INVALID_SOCKET) param->srv->so._getsockname(param->sostate, param->remsock, (struct sockaddr *)&sin,  &sasize);
+	/* CONNECT and BIND report the address of the socket towards the
-	if(command != 3 && !SAISNULL(&param->srv->extNat)){
+	   destination, UDP ASSOCIATE the one the client sends its datagrams
+	   to, as does a request which never got a socket of its own */
+	if(command != 3 && param->remsock != INVALID_SOCKET){
+	    param->srv->so._getsockname(param->sostate, param->remsock, (struct sockaddr *)&sin,  &sasize);
+	    if(!SAISNULL(&param->srv->extNat)){
 		uint16_t port;
 		port = *SAPORT(&sin);
 		sin = param->srv->extNat;
 		*SAPORT(&sin) = port;
 	    }
+	}
 	else {
 	    param->srv->so._getsockname(param->sostate, param->clisock, (struct sockaddr *)&sin,  &sasize);
 	    if(command == 3 && !SAISNULL(&param->srv->intNat)){

src/ssl.c

@@ -779,7 +779,7 @@ static FILTER_ACTION ssl_filter_client(void *fo, struct clientparam * param, voi
 }
 
 static FILTER_ACTION ssl_filter_connect(void *fc, struct clientparam * param){
-	if(PCONF->cli && !client_mode) {
+	if(PCONF->cli && !PCONF->client_mode) {
 	    if(docli(param)) {
 		return REJECT;
 	    }
@@ -788,7 +788,7 @@ static FILTER_ACTION ssl_filter_connect(void *fc, struct clientparam * param){
 }
 
 static FILTER_ACTION ssl_filter_afterauth(void *fc, struct clientparam * param){
-	if(PCONF->cli && client_mode == 1) {
+	if(PCONF->cli && PCONF->client_mode == 1) {
 	    if(docli(param)) {
 		return REJECT;
 	    }
@@ -806,7 +806,7 @@ static FILTER_ACTION ssl_filter_predata(void *fc, struct clientparam * param){
 	    if(!param->redirectfunc) param->redirectfunc = proxyfunc;
 	    return CONTINUE;
 	}
-	else if(PCONF->cli && client_mode == 2) {
+	else if(PCONF->cli && PCONF->client_mode == 2) {
 	    if(docli(param)) {
 		return REJECT;
 	    }
@@ -815,7 +815,7 @@ static FILTER_ACTION ssl_filter_predata(void *fc, struct clientparam * param){
 }
 
 static FILTER_ACTION ssl_parent(struct clientparam * param){
-	if(PCONF->cli && client_mode == 3) {
+	if(PCONF->cli && PCONF->client_mode == 3) {
 	    if(docli(param)) {
 		return REJECT;
 	    }
@@ -867,6 +867,9 @@ static void ssl_filter_close(void *fo){
     free(CONFIG->client_ca_file);
     free(CONFIG->client_ca_dir);
     free(CONFIG->client_ca_store);
+    free(CONFIG->server_ca_file);
+    free(CONFIG->server_ca_dir);
+    free(CONFIG->server_ca_store);
     free(CONFIG->client_sni);
     if(CONFIG->client_alpn_protos.protos_len){
 	free(CONFIG->client_alpn_protos.protos);

src/stringtable.c

@@ -31,8 +31,8 @@ unsigned char * strings[] = {
 /* 18 */	(unsigned char *)"AUTO",
 /* 19 */	(unsigned char *)"TLSPR",
 /* 20 */	(unsigned char *)"IMAPP",
-/* 21 */	(unsigned char *)"ZOMBIE",
+/* 21 */	(unsigned char *)"HTTPSRV",
-/* 22 */	NULL,
+/* 22 */	(unsigned char *)"ZOMBIE",
 /* 23 */	NULL,
 /* 24 */	NULL,
 #ifndef TPROXY_CONF

src/structures.h

@@ -47,6 +47,7 @@ typedef struct _3proxy_sem_s {
 	pthread_cond_t cond;
 	unsigned count;
 	unsigned maxcount;
+	int monotonic;		/* timed waits measured on CLOCK_MONOTONIC */
 } _3proxy_sem_t;
 #else
 #include <winsock2.h>
@@ -528,6 +529,14 @@ struct filterp {
 
 #define MAX_FILTERS 16
 
+/* Socket options for one kind of socket, -oc and the like: which of
+   sockopts[] are given, by their index there, and the value of each. */
+#define MAXSOCKOPTS 32
+struct sockoptset {
+	unsigned set;
+	int value[MAXSOCKOPTS];
+};
+
 
 struct sockfuncs {
 	void * state;
@@ -612,12 +621,13 @@ struct srvparam {
 	int stacksize;
 	int noforce;
 	int anonymous;
-	int clisockopts, srvsockopts, lissockopts, cbcsockopts, cbssockopts;
+	struct sockoptset clisockopts, srvsockopts, lissockopts, cbcsockopts, cbssockopts;
 	int gracetraf, gracenum, gracedelay;
 	int requirecert;
 	int haproxy;
 	int nostarttls;
 	PROXYSERVICE srvstarttls;
+	int allowftp;		/* the http proxy fetches ftp:// for this service */
 	uint32_t fakeip;
 	unsigned char fakeip6[16];
 #ifdef WITHSPLICE
@@ -634,6 +644,7 @@ struct srvparam {
 #endif
 	PROXYSOCKADDRTYPE extsa;
 	_3proxy_mutex_t counter_mutex;
+	_3proxy_sem_t childslots;	/* maxchild of them, one held by each child */
 	struct pollfd fds;
 	FILE *stdlog;
 	unsigned char * target;
@@ -732,6 +743,11 @@ struct clientparam {
 			*clibuf,
 			*srvbuf;
 
+	/* splice() pipes for the client's and the server's data, -1 until
+	   made; kept for the life of the connection */
+	int	pipecli[2],
+		pipesrv[2];
+
 	unsigned 	cliinbuf,
 			srvinbuf,
 			clioffset,
@@ -785,7 +801,6 @@ struct filemon {
 
 
 struct extparam {
-	_3proxy_sem_t threadinit;
 	int *timeouts;
 	struct ace * acl;
 #ifdef WITH_HTTPSRV

tests/README.md

@@ -69,7 +69,7 @@ means to stand on its own should start with one - otherwise an earlier
 
 ## What is not covered yet
 
-41 of the 112 configuration commands appear in a test, and the count says
+42 of the 112 configuration commands appear in a test, and the count says
 nothing about service options: the IPv6 case, for instance, exercises -4,
 -6, -46, -64 and -i without adding a command to it. What follows is
 roughly the order worth working through: how much of the product a gap
@@ -78,12 +78,13 @@ covers, and how much of a fixture it needs.
 ### Traffic limits and accounting
 
 `bandlimin` `bandlimout` `nobandlimin` `nobandlimout` `connlim` `noconnlim`
-`countin` `countout` `countall` and the `no*` forms, `maxconn`.
+`countin` `countout` `countall` and the `no*` forms.
 
 Cheap and worth doing first: `data?size=` and a stopwatch measure a
 bandwidth limit, and the admin counters page already shows what a counter
 holds. `countin` appears in a configuration today but nothing checks that it
-counts. `connlim` and `maxconn` need concurrent connections.
+counts. `connlim` needs concurrent connections, held open the way the
+`maxconn` case holds them.
 
 ### The mail proxies
 

tests/cases/admin.py

@@ -1,10 +1,12 @@
 """The admin interface, now a set of handlers on the HTTP server."""
 
+import time
+
 
 def run(t):
     adm = t.free_port()
     lim = t.free_port()
-    t.start("admin", f"""
+    server = t.start("admin", f"""
         log
         auth iponly
         allow *
@@ -20,17 +22,40 @@ def run(t):
 
     url = f"http://127.0.0.1:{adm}"
 
+    def reload(path):
+        """Ask for a reload and wait for it to be done. It happens on the next
+        tick of the main loop, at most a second later; both old services stop
+        and the new ones start, and a request in between is refused (Windows
+        cannot bind a port still listening) or reset."""
+        before = server.output()
+        accepting = before.count("Accepting connections")
+        exiting = before.count("Exiting thread")
+        reply = t.http(url + path)
+        deadline = time.time() + 10
+        while time.time() < deadline:
+            now = server.output()
+            if (now.count("Accepting connections") >= accepting + 2
+                    and now.count("Exiting thread") >= exiting + 2):
+                break
+            time.sleep(0.05)
+        return reply
+
     # --- the predefined pages -----------------------------------------
     t.eq(200, t.http(url + "/").status, "the main page")
     t.eq(200, t.http(url + "/C").status, "the counters page")
-    t.eq(200, t.http(url + "/R").status, "the reload page")
+    t.eq(200, reload("/R").status, "the reload page")
     t.eq(200, t.http(url + "/S").status, "the services page")
 
     counters = t.http(url + "/C")
     t.contains(counters, "countin", "the counters page names the counter type")
     t.contains(counters, "<tr>", "the counters page renders a table")
-    t.contains(t.http(url + "/R"), "Reload", "the reload page confirms the request")
+    t.contains(reload("/R"), "Reload", "the reload page confirms the request")
     t.contains(t.http(url + "/S"), "<", "the services page returns markup")
+    # admin and httpsrv services are of type HTTPSRV, named as such, not as
+    # the zombie a stopped service is
+    services = t.http(url + "/S")
+    t.contains(services, "CDATA[HTTPSRV]", "the services page names the admin service HTTPSRV")
+    t.not_contains(services, "CDATA[ZOMBIE]", "and no running service a zombie")
 
     # --- the menu no longer offers the removed config editor -----------
     main = t.http(url + "/")

tests/cases/maxconn.py

@@ -0,0 +1,107 @@
+"""maxconn: a service serves at most that many connections at once, and a
+connection over the limit waits for one of them to finish."""
+
+import os
+import signal
+import socket
+import time
+
+
+def run(t):
+    # The origin runs in a process of its own, so the limit only applies to
+    # the mapper in front of it.
+    origin = t.free_port()
+    t.start("maxconn_origin", f"""
+        auth iponly
+        allow *
+        http echo * /echo**
+        httpsrv -p{origin}
+    """, ports=[origin])
+
+    mapped = t.free_port()
+    server = t.start("maxconn", f"""
+        log
+        maxconn 2
+        auth iponly
+        allow *
+        tcppm {mapped} 127.0.0.1 {origin}
+    """, ports=[mapped])
+
+    def connect():
+        sock = socket.create_connection(("127.0.0.1", mapped), 5)
+        opened.append(sock)
+        return sock
+
+    def served(sock, timeout):
+        """Ask over a kept alive connection; True once the whole reply is in,
+        so the connection is ready for the next request."""
+        sock.sendall(b"GET /echo HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n")
+        sock.settimeout(timeout)
+        data = b""
+        try:
+            while b"\r\n\r\n" not in data:
+                chunk = sock.recv(4096)
+                if not chunk:
+                    return False
+                data += chunk
+            head, body = data.split(b"\r\n\r\n", 1)
+            length = 0
+            for line in head.split(b"\r\n")[1:]:
+                name, _, value = line.partition(b":")
+                if name.strip().lower() == b"content-length":
+                    length = int(value)
+            while len(body) < length:
+                chunk = sock.recv(4096)
+                if not chunk:
+                    return False
+                body += chunk
+            return head.startswith(b"HTTP/1.1 200")
+        except OSError:
+            return False
+
+    opened = []
+    try:
+        # the probe that saw the port open holds a slot until its session
+        # ends, so wait for both slots to be free before taking them
+        deadline = time.time() + 5
+        while True:
+            first, second = connect(), connect()
+            both = served(first, 1) and served(second, 1)
+            if both or time.time() > deadline:
+                break
+            first.close()
+            second.close()
+        t.eq(True, both, "two connections are served under maxconn 2")
+
+        third = connect()
+        t.eq(False, served(third, 1.5),
+             "a third connection waits while both slots are held")
+        t.contains(t.wait_output(server, "too many connected clients"),
+                   "too many connected clients",
+                   "a service at its limit says so in the log")
+
+        first.close()
+        t.eq(True, served(third, 5),
+             "the waiting connection is served once a slot is freed")
+        t.eq(True, served(second, 5),
+             "a connection holding a slot keeps being served")
+
+        if os.name == "nt":
+            t.skip("reload while at the limit (no SIGUSR1 on Windows)")
+        else:
+            waiting = connect()
+            t.eq(False, served(waiting, 1),
+                 "the limit holds again with two connections open")
+            server.proc.send_signal(signal.SIGUSR1)
+            # A connection made the instant the signal arrives can land on
+            # the listener being replaced and be reset; give the new service
+            # the moment it needs.
+            time.sleep(1.5)
+            fresh = connect()
+            t.eq(True, served(fresh, 5),
+                 "a reload while the limit is reached starts a service with "
+                 "its slots free")
+            t.eq(None, server.proc.poll(), "the process survives the reload")
+    finally:
+        for sock in opened:
+            sock.close()

tests/cases/portmap.py

@@ -1,5 +1,29 @@
 """The port mappers: tcppm forwards a TCP port, udppm a UDP one."""
 
+import socket
+import threading
+import time
+
+HALFCLOSE_SIZE = 20000000
+
+
+def halfclose_origin(listener):
+    """Answers each connection which asks for something with HALFCLOSE_SIZE
+    bytes, then waits for the client's end of the stream and closes. The
+    connections waiting for the service to start ask nothing."""
+    while True:
+        try:
+            c, _ = listener.accept()
+        except OSError:
+            return
+        try:
+            if c.recv(1000):
+                c.sendall(b"x" * HALFCLOSE_SIZE)
+                c.recv(10)
+        except OSError:
+            pass
+        c.close()
+
 
 def run(t):
     # --- tcppm ---------------------------------------------------------
@@ -40,6 +64,41 @@ def run(t):
 
     t.stop_all()
 
+    # A client which shuts down its sending side still gets the whole answer,
+    # even when it reads slowly: the server has finished and closed by then,
+    # and the rest of the answer waits in the proxy's buffer and the kernel's.
+    listener = socket.socket()
+    listener.bind(("127.0.0.1", 0))
+    listener.listen(1)
+    threading.Thread(target=halfclose_origin, args=(listener,), daemon=True).start()
+    halfclosed = t.free_port()
+    t.start("portmap_halfclose", f"""
+        log
+        auth none
+        tcppm {halfclosed} 127.0.0.1 {listener.getsockname()[1]}
+    """, ports=[halfclosed])
+    s = socket.socket()
+    s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32768)
+    s.settimeout(30)
+    got = 0
+    try:
+        s.connect(("127.0.0.1", halfclosed))
+        s.sendall(b"request")
+        s.shutdown(socket.SHUT_WR)
+        time.sleep(1)
+        while True:
+            d = s.recv(65536)
+            if not d:
+                break
+            got += len(d)
+    except OSError:
+        pass
+    s.close()
+    listener.close()
+    t.eq(HALFCLOSE_SIZE, got, "a half-closed client reading slowly gets the whole answer")
+
+    t.stop_all()
+
     # --- udppm ---------------------------------------------------------
     # something has to be listening for the mapped datagrams to go anywhere
     echo = t.udp_echo()