diff --git a/doc/html/highload.html b/doc/html/highload.html index c4220d4..c2b1be7 100644 --- a/doc/html/highload.html +++ b/doc/html/highload.html @@ -34,7 +34,8 @@ Each connected client requires:
-proxy -osTCP_NODELAY -ocTCP_NODELAY +proxy -osTCP_NODELAY=0 -ocTCP_NODELAY=0-sets TCP_NODELAY for client (oc) and server (os) connections. -
Do not use TCP_NODELAY on slow connections with high delays when -connection bandwidth is a bottleneck. + +
+proxy -s +socks -s +tcppm -s 8080 192.168.1.1 80 ++-s0 turns it off explicitly. In builds without it (CMake option 3PROXY_USE_SPLICE=OFF, +or -DWITHSPLICE removed from the Makefile) -s is accepted and ignored. +
splice() pays off when most of the traffic is large transfers: in the +benchmarks it moved +1.4 to 1.7 times more data than the read/write path at about half the CPU per gigabyte. +It does not help small requests and short connections, which go slower with it, and +every spliced connection needs two more pipes (file descriptors) and more kernel +buffer memory. Data filters are not applied to spliced data. +
TCP_NODELAY and splice() do not conflict and can be combined on high-speed connections.
logdump 1 1is useful +direction. It's especially useful with splice.
logdump 1 1is useful to see how grace delays work; choose a delay value to avoid filling the read buffer (typically 64K) but keep the request sizes close to the chosen average on large file uploads/downloads. diff --git a/man/3proxy.cfg.5 b/man/3proxy.cfg.5 index ffe758b..19668e3 100644 --- a/man/3proxy.cfg.5 +++ b/man/3proxy.cfg.5 @@ -184,8 +184,10 @@ connect to given remote HOST:port instead of listening local connection on -p or .br .B -oc\fIOPTIONS\fB, -os\fIOPTIONS\fB, -ol\fIOPTIONS\fB, -or\fIOPTIONS\fB, -oR\fIOPTIONS\fR options for proxy-to-client (\fB-oc\fR), proxy-to-server (\fB-os\fR), proxy listening (\fB-ol\fR), connect back client (\fB-or\fR), connect back listening (\fB-oR\fR) sockets. +\fIOPTIONS\fR is a comma separated list of \fINAME\fR or \fINAME\fR=\fIVALUE\fR; a name alone sets the option to 1 (TCP_MAXSEG to the value of \fBmaxseg\fR), and \fINAME\fR=0 turns it off. Options add to the defaults and to the options given earlier for the same socket. An unknown name or a value which is not a number is an error. +TCP_NODELAY is set by default on proxy-to-client and proxy-to-server sockets, so a reply written in several parts is not delayed by Nagle's algorithm; use \fB-ocTCP_NODELAY=0\fR or \fB-osTCP_NODELAY=0\fR to turn it off, e.g. to send fewer packets on slow links. Options like TCP_CORK, TCP_NODELAY, TCP_DEFER_ACCEPT, TCP_QUICKACK, TCP_TIMESTAMPS, TCP_FASTOPEN, SO_REUSEADDR, SO_REUSEPORT, SO_EXCLUSIVEADDRUSE, SO_PORT_SCALABILITY, SO_REUSE_UNICASTPORT, SO_KEEPALIVE, SO_DONTROUTE may be supported depending on OS. -SO_REUSEADDR and SO_REUSEPORT are set on the listening socket by default on Unix. On Windows SO_REUSEADDR is not set: it is not required to rebind a listening port and it only lets another local process bind the same address and port. Use SO_EXCLUSIVEADDRUSE (Windows) on the listening socket (\fB-ol\fR) to prevent that. +SO_REUSEADDR and SO_REUSEPORT are set by default on the listening sockets (\fB-ol\fR, \fB-oR\fR) on Unix; \fB-olSO_REUSEPORT=0\fR turns one off. On Windows SO_REUSEADDR is not set: it is not required to rebind a listening port and it only lets another local process bind the same address and port. Use SO_EXCLUSIVEADDRUSE (Windows) on the listening socket (\fB-ol\fR) to prevent that. .br .B -H (for all services) Expect HAProxy PROXY protocol v1 header on incoming connection. @@ -200,11 +202,13 @@ delay GRACE_DELAY milliseconds before polling if average polling size is below G (for admin) secure, allow only secure operations, currently only traffic counters view without ability to reset. .br - (for TCP services, Linux) enable splice(). splice() is not built by default and -is disabled even when built, because current Linux does not implement -SPLICE_F_MOVE, so no real zero-copy takes place and the read/write path is faster -for most traffic. Rebuild with -DWITHSPLICE to make -s available, -s0 disables it -explicitly. + (for TCP services, Linux) use splice() to pass data between the client and the +server without copying it through 3proxy. It is off unless -s is given (-s0 turns +it off explicitly). splice() suits large transfers: they go faster and use less +CPU. Small requests and short connections go slower with it, every connection +needs two more pipes (file descriptors) and more kernel memory, and data filters +are not applied to spliced data. Available in builds with WITHSPLICE, the default +on Linux. .br .B -C (for TCP services) a connection closed by any of the sides terminates the diff --git a/src/common.c b/src/common.c index 10bf296..8fa14bb 100644 --- a/src/common.c +++ b/src/common.c @@ -746,7 +746,7 @@ int doconnect(struct clientparam * param){ param->sinsl = param->srv->extsa; } *SAPORT(¶m->sinsl) = 0; - setopts(param->remsock, param->srv->srvsockopts); + setopts(param->remsock, ¶m->srv->srvsockopts); #ifndef _WIN32 /* not on Windows, see the accepting side in proxymain.c */ diff --git a/src/proxy.h b/src/proxy.h index 7328bc5..19dbccc 100644 --- a/src/proxy.h +++ b/src/proxy.h @@ -501,7 +501,8 @@ extern struct socketoptions { int opt; char * optname; } sockopts[]; -void setopts(SOCKET s, int opts); +int getopts(const char *s, struct sockoptset *opts); +void setopts(SOCKET s, const struct sockoptset *opts); char * printopts(char *sep); #ifdef _WINCE diff --git a/src/proxymain.c b/src/proxymain.c index 6a531bf..53c4283 100644 --- a/src/proxymain.c +++ b/src/proxymain.c @@ -224,27 +224,62 @@ char * printopts(char *sep){ } -int getopts(const char *s){ - int i=0, ret=0; - for(; sockopts[i].optname; i++)if(strstr(s,sockopts[i].optname)) ret |= (1<= MAXSOCKOPTS) return -1; + opts->set |= (1u << i); + opts->value[i] = value; + } + s = *end? end + 1 : end; + } + return 0; } -void setopts(SOCKET s, int opts){ - int i, opt, set; - for(i = 0; opts >= (opt = (1<set & (1u << i))) continue; + value = opts->value[i]; #ifdef TCP_MAXSEG - if(sockopts[i].opt == TCP_MAXSEG){ - if(!conf.maxseg) continue; - set = conf.maxseg; + if(sockopts[i].opt == TCP_MAXSEG && *sockopts[i].optname == 'T'){ + if(!value) value = conf.maxseg; + if(!value) continue; } #endif - if(opts & opt) setsockopt(s, *sockopts[i].optname == 'T'? IPPROTO_TCP: + setsockopt(s, *sockopts[i].optname == 'T'? IPPROTO_TCP: #ifdef SOL_IP *sockopts[i].optname == 'I'? SOL_IP: #endif - SOL_SOCKET, sockopts[i].opt, (char *)&set, sizeof(set)); + SOL_SOCKET, sockopts[i].opt, (char *)&value, sizeof(value)); } } @@ -356,7 +391,9 @@ int MODULEMAINFUNC (int argc, char** argv){ " -64 Prefer IPv6 for outgoing connections, use both IPv4 and IPv6\n" " -ocOPTIONS, -osOPTIONS, -olOPTIONS, -orOPTIONS -oROPTIONS - options for\n" " to-client (oc), to-server (os), listening (ol) socket, connect back client\n" - " (or) socket, connect back server (oR) listening socket\n" + " (or) socket, connect back server (oR) listening socket, a comma separated\n" + " list of NAME or NAME=VALUE (NAME=0 turns an option off); TCP_NODELAY is\n" + " set on to-client and to-server sockets by default\n" " where possible options are: "; #endif @@ -401,6 +438,34 @@ int MODULEMAINFUNC (int argc, char** argv){ srvinit(&srv, &defparam); srv.pf = childdef.pf; isudp = childdef.isudp; + + /* Default socket options, which -oc, -os, -ol and -oR change by naming + them, NAME=0 to turn one off. + + Nagle's algorithm holds back a short write until the previous one is + acknowledged, and a peer delays its acknowledgement, which stalls every + reply written in more than one piece. + + SO_REUSEADDR is not set on Windows: it is not needed to rebind a + listening port there, and it only allows another local process to bind + the same address and port, with undefined behaviour as to which socket + receives the connections. Use -olSO_EXCLUSIVEADDRUSE to prevent that + instead. A Windows build which does not share the listening socket is the + exception: its UDP services bind a second socket to the same address to + answer from, and Windows only allows that when both sockets ask for it. + */ + (void)getopts("TCP_NODELAY", &srv.clisockopts); + (void)getopts("TCP_NODELAY", &srv.srvsockopts); +#ifndef _WIN32 + (void)getopts("SO_REUSEADDR", &srv.lissockopts); + (void)getopts("SO_REUSEADDR", &srv.cbssockopts); +#elif !defined SHARE_UDP_SOCKET + if(isudp) (void)getopts("SO_REUSEADDR", &srv.lissockopts); +#endif +#ifdef SO_REUSEPORT + (void)getopts("SO_REUSEPORT", &srv.lissockopts); + (void)getopts("SO_REUSEPORT", &srv.cbssockopts); +#endif #ifndef NOUDPMAIN if(isudp) { if(!udp_table.ihashtable)inithashtable(&udp_table, 64, 256, 65536); @@ -638,19 +703,19 @@ int MODULEMAINFUNC (int argc, char** argv){ case 'o': switch(argv[i][2]){ case 's': - srv.srvsockopts = getopts(argv[i]+3); + error = getopts(argv[i]+3, &srv.srvsockopts); break; case 'c': - srv.clisockopts = getopts(argv[i]+3); + error = getopts(argv[i]+3, &srv.clisockopts); break; case 'l': - srv.lissockopts = getopts(argv[i]+3); + error = getopts(argv[i]+3, &srv.lissockopts); break; case 'r': - srv.cbcsockopts = getopts(argv[i]+3); + error = getopts(argv[i]+3, &srv.cbcsockopts); break; case 'R': - srv.cbcsockopts = getopts(argv[i]+3); + error = getopts(argv[i]+3, &srv.cbssockopts); break; default: error = 1; @@ -829,42 +894,13 @@ int MODULEMAINFUNC (int argc, char** argv){ perror("socket()"); return -2; } - setopts(sock, srv.lissockopts); + setopts(sock, &srv.lissockopts); #ifdef _WIN32 ioctlsocket(sock, FIONBIO, &ul); #else fcntl(sock,F_SETFL,O_NONBLOCK | fcntl(sock,F_GETFL)); #endif srv.srvsock = sock; -#ifdef WITH_UN - if(*SAFAMILY(&srv.intsa) != AF_UNIX) -#endif - { -/* SO_REUSEADDR is not set on Windows: it is not needed to rebind a listening - port there, and it only allows another local process to bind the same - address and port, with undefined behaviour as to which socket receives the - connections. Use -olSO_EXCLUSIVEADDRUSE to prevent that instead. - - A Windows build which does not share the listening socket is the exception: - its UDP services bind a second socket to the same address to answer from, - and Windows only allows that when both sockets ask for it. - */ -#ifndef _WIN32 - opt = 1; - if(srv.so._setsockopt(srv.so.state, sock, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(int)))perror("setsockopt()"); -#else -#ifndef SHARE_UDP_SOCKET - if(isudp){ - opt = 1; - if(srv.so._setsockopt(srv.so.state, sock, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(int)))perror("setsockopt()"); - } -#endif -#endif -#ifdef SO_REUSEPORT - opt = 1; - srv.so._setsockopt(srv.so.state, sock, SOL_SOCKET, SO_REUSEPORT, (char *)&opt, sizeof(int)); -#endif - } #if defined SO_BINDTODEVICE || defined IP_BOUND_IF if(bindtodevice(&srv.so, srv.so.state, sock, srv.ibindtodevice, *SAFAMILY(&srv.intsa))) { dolog(&defparam, (unsigned char *)"failed to bind device"); @@ -957,16 +993,7 @@ int MODULEMAINFUNC (int argc, char** argv){ freesrvstrings(&srv, cbc_string, cbl_string); return -6; } -#ifndef _WIN32 - opt = 1; - srv.so._setsockopt(srv.so.state, srv.cbsock, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(int)); -#endif -#ifdef SO_REUSEPORT - opt = 1; - srv.so._setsockopt(srv.so.state, srv.cbsock, SOL_SOCKET, SO_REUSEPORT, (char *)&opt, sizeof(int)); -#endif - - setopts(srv.cbsock, srv.cbssockopts); + setopts(srv.cbsock, &srv.cbssockopts); if(srv.so._bind(srv.so.state, srv.cbsock, (struct sockaddr*)&cbsa, SASIZE(&cbsa))==-1) { dolog(&defparam, (unsigned char *)"Failed to bind connect back socket"); @@ -1025,7 +1052,7 @@ int MODULEMAINFUNC (int argc, char** argv){ if(iscbc){ new_sock=so._socket(so.state, SASOCK(&defparam.sincr), SOCK_STREAM, IPPROTO_TCP); if(new_sock != INVALID_SOCKET){ - setopts(new_sock, srv.cbcsockopts); + setopts(new_sock, &srv.cbcsockopts); parsehost(srv.family, cbc_string, (struct sockaddr *)&defparam.sincr); #if defined SO_BINDTODEVICE || defined IP_BOUND_IF @@ -1102,7 +1129,7 @@ int MODULEMAINFUNC (int argc, char** argv){ } continue; } - setopts(new_sock, srv.clisockopts); + setopts(new_sock, &srv.clisockopts); } size = sizeof(defparam.sincl); if(srv.so._getsockname(srv.so.state, new_sock, (struct sockaddr *)&defparam.sincl, &size)){ diff --git a/src/socks.c b/src/socks.c index 5cdc1d3..c39ffab 100644 --- a/src/socks.c +++ b/src/socks.c @@ -184,7 +184,7 @@ void * sockschild(struct clientparam* param) { param->operation = command == 2?BIND:UDPASSOC; if(command == 2){ if ((param->remsock=param->srv->so._socket(param->sostate, SASOCK(¶m->req), SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) {RETURN (11);} - setopts(param->remsock, param->srv->lissockopts); + setopts(param->remsock, ¶m->srv->lissockopts); if(extbindtodevice(param, param->remsock)) {RETURN (12);} } break; diff --git a/src/structures.h b/src/structures.h index 42e6640..472e44a 100644 --- a/src/structures.h +++ b/src/structures.h @@ -529,6 +529,14 @@ struct filterp { #define MAX_FILTERS 16 +/* Socket options for one kind of socket, -oc and the like: which of + sockopts[] are given, by their index there, and the value of each. */ +#define MAXSOCKOPTS 32 +struct sockoptset { + unsigned set; + int value[MAXSOCKOPTS]; +}; + struct sockfuncs { void * state; @@ -613,7 +621,7 @@ struct srvparam { int stacksize; int noforce; int anonymous; - int clisockopts, srvsockopts, lissockopts, cbcsockopts, cbssockopts; + struct sockoptset clisockopts, srvsockopts, lissockopts, cbcsockopts, cbssockopts; int gracetraf, gracenum, gracedelay; int requirecert; int haproxy;