Update HTML documentation from man pages

41 of 112 configuration commands appear in a test. Record the rest as a
plan, ordered by how much of the product each gap covers against how much
of a fixture it needs, so the next person does not have to redo the survey.

Includes the two limitations left deliberately unasserted, and why pinning
the current behaviour there would be wrong.

SECURITY.md

@@ -7,6 +7,25 @@
 | 0.9.8   | :white_check_mark: |
 | < 0.9.8 | :x:                |
 
+## Hardening a deployment
+
+Configuration is where most of the risk lives. The security recommendations are
+kept in [doc/html/securityen.html](doc/html/securityen.html), published at
+<https://3proxy.org/securityen.html>: how to run the service, what the
+ACLs have to cover, and the settings whose defaults are safe only until
+something else is enabled alongside them.
+
+Read it before exposing a service. Recurring points from it:
+
+- Run unprivileged, never suid, and chroot where the platform allows.
+- Name the internal and external interfaces explicitly, and limit sources and
+  destinations with ACLs rather than relying on defaults.
+- Enabling IPv6 makes ACLs written in IPv4 incomplete: the same host is
+  reachable through an IPv4-mapped address, and the IPv6 loopback is an
+  address of its own.
+- Anything that terminates or intercepts TLS holds key material and sees full
+  request URLs; both the key and the logs need protecting.
+
 ## Reporting a Vulnerability
 
 Report to 3proxy@3proxy.org or via [GitHub security reporting](https://github.com/3proxy/3proxy/security)

doc/html/howtoe.html

@@ -828,12 +828,32 @@ This creates an HTTPS proxy (ssl_serv) that accepts TLS connections from clients
 # Generate CA private key
 openssl genrsa -out ca.key 4096
 
+# Extensions that make the certificate usable as a CA
+cat > ca.ext << 'EOF'
+basicConstraints=critical,CA:TRUE
+keyUsage=critical,keyCertSign,cRLSign
+subjectKeyIdentifier=hash
+EOF
+
 &#35; Generate CA certificate (valid for 10 years)
-openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 \
+openssl req -new -nodes -key ca.key \
     -subj "/C=US/ST=State/L=City/O=MyOrg/CN=My CA" \
-    -out ca.crt
+    -out ca.csr
+openssl x509 -req -in ca.csr -signkey ca.key -sha256 -days 3650 \
+    -extfile ca.ext -out ca.crt
 </pre>
 <p>
+The extensions are not optional. Without <b>basicConstraints=CA:TRUE</b> and
+<b>keyCertSign</b> the certificate is not accepted as a CA, and clients report
+that they cannot get the local issuer certificate. <b>subjectKeyIdentifier</b>
+is what certificates signed by this CA point back at.
+</p>
+<p>
+They are given in a file rather than with <b>-addext</b> because LibreSSL, the
+<b>openssl</b> command on macOS and some BSDs, does not apply -addext the same
+way OpenSSL does. The form above behaves the same on both.
+</p>
+<p>
 For MITM, import ca.crt into client browsers/OS as a trusted root CA.
 </p>
 <p>
@@ -866,8 +886,18 @@ EOF
 openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key \
     -CAcreateserial -out server.crt -days 365 -sha256 \
     -extfile server.ext
+
+&#35; Check it the way a current client will
+openssl verify -x509_strict -CAfile ca.crt server.crt
 </pre>
 <p>
+Verify strictly, because that is what the client does. OpenSSL 3 adds the
+subject and authority key identifiers when it signs and LibreSSL does not,
+which is why the extensions file asks for them by name. Python has verified
+strictly since 3.13 and refuses a certificate carrying no
+<b>authorityKeyIdentifier</b>; other clients are moving the same way.
+</p>
+<p>
 For a public https:// proxy, use a CA like Let's Encrypt instead of self-signed.
 </p>
 <p>
@@ -886,6 +916,8 @@ cat > client.ext << 'EOF'
 basicConstraints=CA:FALSE
 keyUsage = digitalSignature, nonRepudiation, keyEncipherment
 extendedKeyUsage = clientAuth
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
 EOF
 
 &#35; Sign with CA
@@ -908,8 +940,14 @@ Import client1.p12 into the client browser or OS certificate store.
 
 &#35; CA
 openssl genrsa -out ca.key 4096
-openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 \
+cat > ca.ext << 'EOF'
-    -subj "/CN=3proxy CA" -out ca.crt
+basicConstraints=critical,CA:TRUE
+keyUsage=critical,keyCertSign,cRLSign
+subjectKeyIdentifier=hash
+EOF
+openssl req -new -nodes -key ca.key -subj "/CN=3proxy CA" -out ca.csr
+openssl x509 -req -in ca.csr -signkey ca.key -sha256 -days 3650 \
+    -extfile ca.ext -out ca.crt
 
 &#35; Server
 openssl genrsa -out server.key 2048
@@ -919,6 +957,8 @@ basicConstraints=CA:FALSE
 keyUsage = keyEncipherment
 extendedKeyUsage = serverAuth
 subjectAltName = DNS:localhost,DNS:proxy,IP:127.0.0.1
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
 EOF
 openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key \
     -CAcreateserial -out server.crt -days 365 -sha256 -extfile server.ext
@@ -929,11 +969,17 @@ openssl req -new -key client.key -subj "/CN=client" -out client.csr
 cat > client.ext << 'EOF'
 basicConstraints=CA:FALSE
 extendedKeyUsage = clientAuth
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
 EOF
 openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key \
     -CAcreateserial -out client.crt -days 365 -sha256 -extfile client.ext
 openssl pkcs12 -export -out client.p12 -passout pass: \
     -inkey client.key -in client.crt -certfile ca.crt
+
+&#35; Both must pass the checks a current client applies
+openssl verify -x509_strict -CAfile ca.crt server.crt
+openssl verify -x509_strict -CAfile ca.crt client.crt
 </pre>
  <li><a name="PCRE"><i>How to use PCRE filtering (regular expressions)</i></a>
 <p>

doc/html/howtor.html

@@ -838,12 +838,32 @@ ssl_nocli
 # Генерация закрытого ключа CA
 openssl genrsa -out ca.key 4096
 
+# Расширения, без которых сертификат не годится как CA
+cat > ca.ext << 'EOF'
+basicConstraints=critical,CA:TRUE
+keyUsage=critical,keyCertSign,cRLSign
+subjectKeyIdentifier=hash
+EOF
+
 &#35; Генерация сертификата CA (действителен 10 лет)
-openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 \
+openssl req -new -nodes -key ca.key \
     -subj "/C=RU/ST=Region/L=City/O=MyOrg/CN=My CA" \
-    -out ca.crt
+    -out ca.csr
+openssl x509 -req -in ca.csr -signkey ca.key -sha256 -days 3650 \
+    -extfile ca.ext -out ca.crt
 </pre>
 <p>
+Расширения обязательны. Без <b>basicConstraints=CA:TRUE</b> и
+<b>keyCertSign</b> сертификат не принимается как CA, и клиент сообщает, что не
+может получить сертификат издателя. <b>subjectKeyIdentifier</b> — то, на что
+ссылаются подписанные этим CA сертификаты.
+</p>
+<p>
+Расширения задаются файлом, а не через <b>-addext</b>, потому что LibreSSL —
+команда <b>openssl</b> в macOS и некоторых BSD — обрабатывает -addext иначе,
+чем OpenSSL. Приведённый вариант одинаково работает в обоих.
+</p>
+<p>
 Для MITM импортируйте ca.crt в браузеры/ОС клиентов как доверенный корневой CA.
 </p>
 <p>
@@ -876,8 +896,18 @@ EOF
 openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key \
     -CAcreateserial -out server.crt -days 365 -sha256 \
     -extfile server.ext
+
+&#35; Проверка так же, как это делает современный клиент
+openssl verify -x509_strict -CAfile ca.crt server.crt
 </pre>
 <p>
+Проверять следует строго, потому что именно так проверяет клиент. OpenSSL 3
+добавляет идентификаторы ключей при подписании, а LibreSSL — нет, поэтому файл
+расширений запрашивает их явно. Python начиная с 3.13 проверяет строго и
+отвергает сертификат без <b>authorityKeyIdentifier</b>; другие клиенты идут тем
+же путём.
+</p>
+<p>
 Для публичного https:// прокси используйте CA вроде Let's Encrypt вместо самоподписанного.
 </p>
 <p>
@@ -896,6 +926,8 @@ cat > client.ext << 'EOF'
 basicConstraints=CA:FALSE
 keyUsage = digitalSignature, nonRepudiation, keyEncipherment
 extendedKeyUsage = clientAuth
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
 EOF
 
 &#35; Подписание CA
@@ -918,8 +950,14 @@ openssl pkcs12 -export -out client1.p12 \
 
 &#35; CA
 openssl genrsa -out ca.key 4096
-openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 \
+cat > ca.ext << 'EOF'
-    -subj "/CN=3proxy CA" -out ca.crt
+basicConstraints=critical,CA:TRUE
+keyUsage=critical,keyCertSign,cRLSign
+subjectKeyIdentifier=hash
+EOF
+openssl req -new -nodes -key ca.key -subj "/CN=3proxy CA" -out ca.csr
+openssl x509 -req -in ca.csr -signkey ca.key -sha256 -days 3650 \
+    -extfile ca.ext -out ca.crt
 
 &#35; Сервер
 openssl genrsa -out server.key 2048
@@ -929,6 +967,8 @@ basicConstraints=CA:FALSE
 keyUsage = keyEncipherment
 extendedKeyUsage = serverAuth
 subjectAltName = DNS:localhost,DNS:proxy,IP:127.0.0.1
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
 EOF
 openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key \
     -CAcreateserial -out server.crt -days 365 -sha256 -extfile server.ext
@@ -939,11 +979,17 @@ openssl req -new -key client.key -subj "/CN=client" -out client.csr
 cat > client.ext << 'EOF'
 basicConstraints=CA:FALSE
 extendedKeyUsage = clientAuth
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
 EOF
 openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key \
     -CAcreateserial -out client.crt -days 365 -sha256 -extfile client.ext
 openssl pkcs12 -export -out client.p12 -passout pass: \
     -inkey client.key -in client.crt -certfile ca.crt
+
+&#35; Оба должны пройти проверку, которую делает современный клиент
+openssl verify -x509_strict -CAfile ca.crt server.crt
+openssl verify -x509_strict -CAfile ca.crt client.crt
 </pre>
 
  <li><a name="PCRE"><i>Как использовать PCRE-фильтрацию (регулярные выражения)</i></a>

doc/html/man5/3proxy.cfg.5.html

@@ -124,7 +124,9 @@ udppm</b> UDP portmapper</p>
 <b><br>
 -6</b> Only resolve IPv6 addresses. IPv4 addresses are
 packed in IPv6 in IPV6_V6ONLY compatible way. <b><br>
--4</b> Only resolve IPv4 addresses <b><br>
+-4</b> Only resolve IPv4 addresses. This is the default: a
+service reaches an IPv6 address only when told to with
+<b>-6</b>, <b>-46</b> or <b>-64</b>. <b><br>
 -46</b> Prefer IPv4. Resolve IPv6 addresses if IPv4 address
 is not resolvable <b><br>
 -64</b> Prefer IPv6. Resolve IPv4 addresses if IPv6 address
@@ -495,14 +497,23 @@ the same as for nserver.</p>
 Cache <i>&lt;cachesize&gt;</i> records for name resolution
 (<b>nscache</b> for IPv4, <b>nscache6</b> for IPv6). The
 cache size should usually be large enough (for example,
-65536).</p>
+65536). The two are separate: a name that resolves to an
+IPv6 address, including one given with <b>nsrecord</b>, is
+only held when <b>nscache6</b> is configured, and
+<b>nscache</b> does nothing for it. Both caches are global
+rather than per-service.</p>
 
 <p style="margin-left:9%; margin-top: 1em"><b>nsrecord</b>
 <i>&lt;hostname&gt; &lt;hostaddr&gt;</i> <br>
 Adds static record to nscache. <b>nscache</b> must be
-enabled. If 0.0.0.0 is used as a hostaddr host will never
+enabled and must come first, because the record is placed in
-resolve, it can be used to blacklist something or together
+the table it allocates - <b>nscache6</b> for a record naming
-with <b>dialer</b> command to set up UDL for dialing.</p>
+an IPv6 address - and <b>nserver</b> must be set as well:
+without it the system resolver is used and static records
+are never consulted. If 0.0.0.0 is used as a hostaddr host
+will never resolve, it can be used to blacklist something or
+together with <b>dialer</b> command to set up UDL for
+dialing.</p>
 
 
 <p style="margin-left:9%; margin-top: 1em"><b>fakeresolve</b>
@@ -1344,7 +1355,7 @@ Apply a rule for matching regular expression. <b><br>
 pcre_rewrite</b> <i>TYPE FILTER_ACTION REGEXP
 REWRITE_EXPRESSION [ACE]</i> <br>
 Match and replace with rewrite expression. <b><br>
-pcre_extend</b> <i>FILTER_ACTION [ACE]</i> <br>
+pcre_extend</b> <i>ACE</i> <br>
 Extend the ACL of the last pcre or pcre_rewrite command by
 adding an additional ACE. <b><br>
 pcre_options</b> <i>OPTION1 [OPTION2 ...]</i> <br>
@@ -1403,7 +1414,14 @@ required.</p>
 - substitution string. May contain Perl-style substrings $1,
 $2, etc. $0 means the whole matched string. \r and \n may be
 used to insert new lines; the string may be empty
-(&quot;&quot;).</p>
+(&quot;&quot;). <br>
+A rewritten request is what the server receives. The
+destination is chosen, and the access rules are applied to
+it, before the filters run, so a rewrite that names another
+host or changes the method is logged but not acted on: the
+request is still sent where the access rules allowed.
+Rewriting the path or the query works on a direct connection
+and through a parent alike.</p>
 
 <p style="margin-left:9%; margin-top: 1em">ACE - access
 control entry (user names, source IPs, destination IPs,

doc/html/securityen.html

@@ -19,6 +19,45 @@ authentication is currently available.
 <li>Always limit connections to the internal network and localhost (to 127.0.0.1 and
 all interfaces) with ACLs. Be careful, because the BIND command in SOCKS requires the
 BIND method with the external interface IP address to be allowed.
+<li>Services resolve IPv4 only unless told otherwise ('-4' is the default). Enabling
+IPv6 with '-6', '-46' or '-64' makes every ACL written in IPv4 incomplete, because the
+same host can be asked for in another way. A proxy that denies 127.0.0.1 but has IPv6
+enabled still reaches that host as '::ffff:127.0.0.1', and reaches the machine again as
+'::1', which is a different address the IPv4 rule never mentioned. When IPv6 is enabled,
+deny the mapped form '::ffff:0:0/96' as well unless it is needed, and deny the IPv6
+addresses that correspond to whatever the IPv4 rules protect: '::1' and '::' for the
+local machine, 'fe80::/10' for link-local and 'fc00::/7' for unique local addresses.
+Denying the IPv4 spelling alone is not enough.
+<li>With '-46' or '-64' a name resolves to either family, so a target ACL that names
+only one of a host's addresses does not limit that host. Names are resolved into
+separate caches, and a name that resolves to an IPv6 address is only cached when
+'nscache6' is configured.
+<li>The 'admin' service hands out counters, the list of running services and a way to
+trigger a configuration reload. Bind it to an internal interface, and put
+authentication and an ACL in front of it. The '-s' option limits what the pages offer
+but is not authentication.
+<li>The 'echo' and 'data' operations of the 'http' command exist for testing. 'data'
+returns a response of whatever size the request asks for, so a listener offering it to
+anyone is a traffic amplifier. Do not configure them on a public service.
+<li>'ssl_server_ca_key' is the private key of a certificate authority that clients have
+been told to trust. Anyone who obtains it can impersonate any site to those clients, so
+protect it as a signing key and use a CA created for this purpose only, never one that
+is trusted for anything else. Restrict the 'ssl_certcache' directory as well: it holds
+the certificates generated from that key.
+<li>Interception ('ssl_mitm') ends the guarantee the client believes it has. The full
+URL of every request inside the tunnel, query string included, becomes visible to the
+proxy and reaches the log, where a plain CONNECT would have shown only a host and a
+port. Treat those logs accordingly.
+<li>Certificates generated for interception by a build against wolfSSL carry no key
+identifiers, because that library cannot generate certificate extensions, and a client
+verifying strictly (OpenSSL 'x509_strict', which recent Python enables by default)
+rejects them. Builds against OpenSSL generate them. Where they are missing, turning
+verification off in the client removes the protection interception was supposed to
+preserve; use an OpenSSL build instead.
+<li>Regular expression rules ('pcre', 'pcre_rewrite') are matched without
+authentication and do not replace ACLs. A rewrite that would change the method or the
+destination of a request is ignored, because the destination was already authorized;
+do not rely on one to redirect traffic.
 <li>Before 3proxy 0.8, always use nserver and nscache under Unix; otherwise, a DoS attack is possible
 with an unreachable DNS server (because gethostbyname will block other threads).
 <li>Keep logs in a secure location, because some confidential information from

man/3proxy.cfg.5

@@ -132,7 +132,8 @@ change default server port to NUMBER
 Only resolve IPv6 addresses. IPv4 addresses are packed in IPv6 in IPV6_V6ONLY compatible way.
 .br
 .B -4
-Only resolve IPv4 addresses
+Only resolve IPv4 addresses. This is the default: a service reaches an IPv6
+address only when told to with \fB-6\fR, \fB-46\fR or \fB-64\fR.
 .br
 .B -46
 Prefer IPv4. Resolve IPv6 addresses if IPv4 address is not resolvable
@@ -521,13 +522,20 @@ If not specified, nserver is used. The syntax is the same as for nserver.
 .br
  Cache \fI<cachesize>\fR records for name resolution (\fBnscache\fR for IPv4,
 \fBnscache6\fR for IPv6). The cache size should usually be large enough
-(for example, 65536).
+(for example, 65536). The two are separate: a name that resolves to an IPv6
+address, including one given with \fBnsrecord\fR, is only held when
+\fBnscache6\fR is configured, and \fBnscache\fR does nothing for it. Both
+caches are global rather than per-service.
 
 .br
 .BR nsrecord
 \fI<hostname>\fR \fI<hostaddr>\fR
 .br
- Adds static record to nscache. \fBnscache\fR must be enabled. If 0.0.0.0
+ Adds static record to nscache. \fBnscache\fR must be enabled and must come
+first, because the record is placed in the table it allocates - \fBnscache6\fR
+for a record naming an IPv6 address - and
+\fBnserver\fR must be set as well: without it the system resolver is used and
+static records are never consulted. If 0.0.0.0
 is used as a hostaddr host will never resolve, it can be used to
 blacklist something or together with 
 .B dialer
@@ -1432,7 +1440,7 @@ Apply a rule for matching regular expression.
 Match and replace with rewrite expression.
 .br
 .BR pcre_extend
-\fIFILTER_ACTION [ACE]\fR
+\fIACE\fR
 .br
 Extend the ACL of the last pcre or pcre_rewrite command by adding an additional ACE.
 .br
@@ -1482,6 +1490,12 @@ REGEXP - PCRE (Perl) regular expression. Use * if no regexp matching is required
 REWRITE_EXPRESSION - substitution string. May contain Perl-style substrings
 $1, $2, etc. $0 means the whole matched string. \er and \en may be used
 to insert new lines; the string may be empty ("").
+.br
+A rewritten request is what the server receives. The destination is chosen,
+and the access rules are applied to it, before the filters run, so a rewrite
+that names another host or changes the method is logged but not acted on:
+the request is still sent where the access rules allowed. Rewriting the path
+or the query works on a direct connection and through a parent alike.
 
 ACE - access control entry (user names, source IPs, destination IPs, ports, etc.),
 identical to allow/deny/bandlimin commands. The regular expression is only

src/httpsrv.c

@@ -27,6 +27,7 @@
 #define HTTPSRV_LINE	1024
 #define HTTPSRV_BLOCK	8192
 #define HTTPSRV_MAXHDR	64
+#define HTTPSRV_MAXBODY	1048576
 
 
 /* Returns the value of a query parameter, or def when it is missing or not a
@@ -363,6 +364,28 @@ int httpopbyname(const unsigned char *name)
 	return -1;
 }
 
+/* Read and discard a request body.
+
+   The reply is followed by a close, and closing a socket that still holds
+   unread data resets the connection rather than ending it, which costs the
+   client the reply it was about to read. Bounded, so a client cannot keep
+   the server reading.
+ */
+static void httpsrv_drain(struct clientparam *param, unsigned long len)
+{
+	char buf[HTTPSRV_BLOCK];
+
+	if(len > HTTPSRV_MAXBODY) len = HTTPSRV_MAXBODY;
+	while(len){
+		int want = (len > (unsigned long)sizeof(buf))? (int)sizeof(buf) : (int)len;
+		int got = sockgetlinebuf(param, CLIENT, (unsigned char *)buf, want, EOF,
+			conf.timeouts[STRING_S]);
+
+		if(got <= 0) break;
+		len -= (unsigned long)got;
+	}
+}
+
 void * httpsrvchild(struct clientparam *param)
 {
 	struct httpreq r;
@@ -451,6 +474,8 @@ void * httpsrvchild(struct clientparam *param)
 		}
 	}
 
+	if(r.contentlen) httpsrv_drain(param, r.contentlen);
+
 	if(r.host[0]){
 		char host[sizeof(r.host)];
 		char *colon;

src/pcre.c

@@ -277,7 +277,7 @@ static FILTER_ACTION pcre_filter_buffer(void *fc, struct clientparam *param, uns
 #define pcrefd ((struct pcre_filter_data *)fc)
 
 	for(acl = pcrefd->acl; acl; acl=acl->next){
-		if(pl->ACLMatches(pcrefd->acl, param)){
+		if(pl->ACLMatches(acl, param)){
 			match = 1;
 			break;
 		}

src/proxy.c

@@ -156,6 +156,26 @@ static void freeptr(void *p){
 	if(*pp) { free(*pp); *pp = NULL; }
 }
 
+#ifndef WITHMAIN
+/* Point at the path in a request line and report the authority it names.
+   Returns NULL if the line is not one we can put back together. */
+static unsigned char * reqpath(unsigned char *line, unsigned char **host, int *hostlen)
+{
+	unsigned char *sp, *p;
+
+	*host = NULL;
+	*hostlen = 0;
+	if(!line || !(sp = (unsigned char *)strchr((char *)line, ' '))) return NULL;
+	while(*sp == ' ') sp++;
+	if(*sp == '/') return sp;
+	if(strncasecmp((char *)sp, "http://", 7)) return NULL;
+	*host = p = sp + 7;
+	while(*p && *p != '/' && *p != ' ') p++;
+	*hostlen = (int)(p - *host);
+	return (*p == '/')? p : NULL;
+}
+#endif
+
 static void logurl(struct clientparam * param, char * buf, char * req, int ftp){
  char *sb;
  char *se;
@@ -254,6 +274,7 @@ void * proxychild(struct clientparam* param) {
  int sleeptime = 0;
 #ifndef WITHMAIN
  int reqsize, reqbufsize;
+ unsigned char *origreq = NULL;
 #endif
  int authenticate;
  struct pollfd fds[2];
@@ -577,11 +598,51 @@ for(;;){
 
 #ifndef WITHMAIN
 
+ /* Only worth keeping a copy when something can rewrite it. */
+ if(param->nreqfilters) origreq = (unsigned char *)strdup((char *)req);
  action = handlereqfilters(param, &req, &reqbufsize, 0, &reqsize);
  if(action == HANDLED){
+	freeptr(&origreq);
 	RETURN(0);
  }
- if(action != PASS) RETURN(517);
+ if(action != PASS){
+	freeptr(&origreq);
+	RETURN(517);
+ }
+
+ /* Only the copy in req was rewritten. On a direct connection the server is
+    sent the request line held in buf, which was parsed and reduced to its
+    path before the filters ran, so put the new path there as well.
+
+    The destination was chosen, and the access rules applied to it, before
+    the rewrite happened. A rewrite that changes the method or the authority
+    is therefore left alone: acting on it would send the request somewhere
+    the rules never saw. */
+ if(origreq && !isconnect && !ftp && strcmp((char *)req, (char *)origreq)){
+	unsigned char *oldhost, *newhost, *oldpath, *newpath;
+	int oldhostlen, newhostlen, methodlen;
+
+	methodlen = (int)(strchr((char *)origreq, ' ') - (char *)origreq);
+	oldpath = reqpath(origreq, &oldhost, &oldhostlen);
+	newpath = reqpath(req, &newhost, &newhostlen);
+	if(oldpath && newpath
+	   && methodlen > 0 && !strncmp((char *)req, (char *)origreq, methodlen)
+	   && req[methodlen] == ' '
+	   && oldhostlen == newhostlen
+	   && (!oldhostlen || !strncasecmp((char *)oldhost, (char *)newhost, oldhostlen))){
+		int newlen = (int)strlen((char *)newpath);
+		int delta = newlen - ((int)reqlen - ssoff);
+
+		if(ssoff > 0 && (int)reqlen >= ssoff && inbuf + delta < bufsize - 1){
+			memmove(buf + ssoff + newlen, buf + reqlen, inbuf - reqlen + 1);
+			memcpy(buf + ssoff, newpath, newlen);
+			inbuf += delta;
+			reqlen += delta;
+			buf[inbuf] = 0;
+		}
+	}
+ }
+ freeptr(&origreq);
  action = handlehdrfilterscli(param, &buf, &bufsize, 0, &inbuf);
  if(action == HANDLED){
 	RETURN(0);

src/ssllib.c

@@ -84,7 +84,11 @@ static int copy_ext(X509 *dst_cert, X509 *src_cert, int nid)
 }
 
 #ifndef WITH_WOLFSSL
-static int add_ext(X509 *cert, int nid, const char *value)
+/* issuer is the certificate the extension should describe as the issuer,
+ * which matters for an authority key identifier: it names the key that
+ * signs, not the key being signed.
+ */
+static int add_ext_issuer(X509 *cert, X509 *issuer, int nid, const char *value)
 {
 	X509_EXTENSION *ex;
 	X509V3_CTX ctx;
@@ -92,10 +96,8 @@ static int add_ext(X509 *cert, int nid, const char *value)
 	/* This sets the 'context' of the extensions. */
 	/* No configuration database */
 	X509V3_set_ctx_nodb(&ctx);
-	/* Issuer and subject certs: both the target since it is self signed,
+	/* No request and no CRL */
-	 * no request and no CRL
+	X509V3_set_ctx(&ctx, issuer, cert, NULL, NULL, 0);
-	 */
-	X509V3_set_ctx(&ctx, cert, cert, NULL, NULL, 0);
 	/* value is char * prior to OpenSSL 1.1.0 */
 	ex = X509V3_EXT_conf_nid(NULL, &ctx, nid, (char *)value);
 	if (!ex)
@@ -105,6 +107,12 @@ static int add_ext(X509 *cert, int nid, const char *value)
 	X509_EXTENSION_free(ex);
 	return err > 0;
 }
+
+static int add_ext(X509 *cert, int nid, const char *value)
+{
+	/* Issuer and subject: both the target, for a self signed certificate */
+	return add_ext_issuer(cert, cert, nid, value);
+}
 #endif
 
 SSL_CERT ssl_copy_cert(SSL_CERT cert, SSL_CONFIG *config)
@@ -199,6 +207,16 @@ SSL_CERT ssl_copy_cert(SSL_CERT cert, SSL_CONFIG *config)
 		add_ext(dst_cert, NID_basic_constraints, "critical,CA:FALSE");
 	if(!copy_ext(dst_cert, src_cert, NID_ext_key_usage))
 		add_ext(dst_cert, NID_ext_key_usage, "serverAuth");
+	/* A verifier following RFC 5280 strictly looks for the issuer through a
+	 * key identifier and refuses a certificate carrying none: OpenSSL does
+	 * with x509_strict, and Python has since 3.13. The identifiers are
+	 * generated rather than copied, so they name the CA signing here
+	 * instead of the one that signed upstream. keyid,issuer keeps working
+	 * when the CA certificate has no subject key identifier of its own.
+	 */
+	add_ext(dst_cert, NID_subject_key_identifier, "hash");
+	add_ext_issuer(dst_cert, config->CA_cert, NID_authority_key_identifier,
+		"keyid,issuer");
 #else
 	copy_ext(dst_cert, src_cert, NID_basic_constraints);
 	copy_ext(dst_cert, src_cert, NID_ext_key_usage);

tests/README.md

@@ -7,7 +7,9 @@
     python3 tests/run.py --keep              # keep the configurations and logs
 
 Python 3.6 or later and a built 3proxy are the only requirements: the suite
-is standard library throughout, so it runs wherever 3proxy builds. With no
+is standard library throughout, so it runs wherever 3proxy builds. The TLS
+case additionally wants `openssl` on PATH to generate its key material, and
+skips itself when that is missing or the build has no TLS support. With no
 `--bin` it looks in `bin/`, then `build/bin/`, then the per-configuration
 directories a multi-configuration CMake generator uses.
 
@@ -49,6 +51,114 @@ Assertions are `eq`, `ne`, `contains`, `not_contains`, `in_range`,
 `not_in_range`, plus `ok`, `fail` and `skip`. `harness.field()` and
 `int_field()` pull a single line out of an `echo` reply.
 
+For services with no TCP port to connect to, `t.udp_echo()` starts an echo
+server, `t.udp_exchange()` sends a datagram, `t.wait_udp()` waits for a UDP
+service to start answering, `t.socks_udp()` carries one through a SOCKS
+association, and `t.dns_query()` asks a DNS server for an A record.
+
+`t.certs()` generates a CA, a second unrelated CA, and a certificate for
+127.0.0.1, once per run and inside the run's temporary directory, so no key
+material lives in the tree. `t.https()`, `t.tls_proxy_http()` and
+`t.socks_http()` reach a server through TLS, a TLS-wrapped proxy, or SOCKS.
+Log records are written when a connection finishes rather than when the
+reply arrives, so assert on them through `t.wait_output(server, text)`.
+
 Note that access rules accumulate until `flush`, so a service section that
 means to stand on its own should start with one - otherwise an earlier
 `allow *` matches first and the rule under test is never reached.
+
+## What is not covered yet
+
+41 of the 112 configuration commands appear in a test, and the count says
+nothing about service options: the IPv6 case, for instance, exercises -4,
+-6, -46, -64 and -i without adding a command to it. What follows is
+roughly the order worth working through: how much of the product a gap
+covers, and how much of a fixture it needs.
+
+### Traffic limits and accounting
+
+`bandlimin` `bandlimout` `nobandlimin` `nobandlimout` `connlim` `noconnlim`
+`countin` `countout` `countall` and the `no*` forms, `maxconn`.
+
+Cheap and worth doing first: `data?size=` and a stopwatch measure a
+bandwidth limit, and the admin counters page already shows what a counter
+holds. `countin` appears in a configuration today but nothing checks that it
+counts. `connlim` and `maxconn` need concurrent connections.
+
+### The mail proxies
+
+`pop3p` `smtpp` `imapp`, and `ftppr`.
+
+The largest gap by volume: four protocol implementations with no coverage at
+all. Each needs a scripted server that speaks enough of the protocol,
+including the multi-line and challenge forms - a POP3 or IMAP server that
+only answers `+OK` will not exercise the interesting paths. Worth the
+fixture: this is also where known parent-chaining trouble lives, since
+`clientnegotiate()` has no case for R_POP3, R_SMTP or R_FTP.
+
+### Access rules and chaining
+
+`redirect` `weight` `parentretries` `force` `noforce` `include` `nolog`.
+
+Also the parts of an ACE never exercised: source addresses and masks, port
+ranges, time and weekday fields, and operation lists beyond the single
+`HTTP_CONNECT` used today. `weight` needs several parents and enough
+requests to see the split.
+
+### IPv6, what is left of it
+
+`tests/cases/ipv6.py` covers listening on `::1`, proxying to and from it,
+SOCKS with an IPv6 destination, rules naming an IPv6 address, and which
+family each of `-4 -6 -46 -64` will use. Still open: `extip` with an IPv6 CIDR, whose
+randomisation path has no coverage.
+
+### Authentication
+
+`authcache` `radius` `authnserver`, and the auth methods beyond `iponly` and
+`strong`: `none`, `nbname`, `dnsname`. `radius` needs a server to answer.
+
+### Plugins
+
+`plugin`. Nothing loads one, though `StringsPlugin`, `TrafficPlugin`,
+`TransparentPlugin` and `FilePlugin` are built in CI. StringsPlugin matters
+most: the admin string table is kept byte-compatible for it deliberately,
+and nothing proves that.
+
+### Logging
+
+`logformat` `rotate` `archiver` `logdump`.
+
+Tests read the log as free text, so a reordered field would pass every check
+here and break every downstream parser. `rotate` and `archiver` need control
+of the clock or a long run.
+
+### TLS options
+
+About 25 `ssl_client_*` and `ssl_server_*` commands: SNI, ALPN, protocol
+versions, cipher lists, `ssl_client_cert` and `ssl_client_key` for mTLS,
+`ssl_*_verify` and `ssl_*_no_verify`. The certificate fixture exists, so
+these are mostly a matter of writing them.
+
+### Process and lifecycle
+
+`daemon` `chroot` `setuid` `setgid` `pidfile` `stacksize` `backlog` `monitor`
+`system` `include` `timeouts` `maxseg` `external` `delimchar`
+`filtermaxsize`. Several need root or change the process in ways a test
+runner has to survive; `include`, `timeouts` and `pidfile` do not, and are
+easy.
+
+Reload is worth a case of its own: the admin page returns "Reload scheduled"
+and nothing checks that the configuration is re-read, that a changed rule
+takes effect, or that services come back.
+
+### DNS
+
+`fakeresolve` `nscache6` `dialer`.
+
+### Known limitations, deliberately not asserted
+
+A request rewrite that changes the method or the authority is ignored, and
+the manual says so; a test that pinned the current behaviour would have to
+change when that does. An intercepted certificate is verified strictly where the build can
+generate the key identifiers, and the case skips that one check on a wolfSSL
+build, which cannot. If wolfSSL gains the ability, the skip should go.

tests/cases/auto.py

@@ -0,0 +1,74 @@
+"""auto: one port that works out which protocol the client is speaking.
+
+Two origins, because the protocols reach different places: an HTTP or SOCKS
+client names its own destination, while a TLS client names a host in the
+handshake and the service supplies the port.
+"""
+
+
+def run(t):
+    certs = t.certs()
+    plain = t.free_port()
+    port = t.free_port()
+    secure = t.free_port() if certs else None
+
+    tls_origin = ""
+    if certs:
+        tls_origin = f"""
+        flush
+        ssl_server_cert {certs.server}
+        ssl_server_key {certs.server_key}
+        ssl_serv
+        auth iponly
+        allow *
+        http * /echo* echo
+        httpsrv -p{secure}
+        ssl_noserv"""
+
+    ports = [plain, port] + ([secure] if certs else [])
+    server = t.start("auto", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        httpsrv -p{plain}
+{tls_origin}
+
+        flush
+        nserver 127.0.0.1
+        nscache 1024
+        nsrecord sni.test 127.0.0.1
+        auth iponly
+        allow *
+        auto -p{port}{f' -P{secure}' if certs else ''}
+    """, ports=ports)
+
+    url = f"http://127.0.0.1:{plain}/echo"
+    at = f"127.0.0.1:{port}"
+
+    # --- as an HTTP proxy -------------------------------------------------
+    r = t.http(url, proxy=at)
+    t.eq(200, r.status, "the same port serves an HTTP proxy request")
+    t.contains(r, "path=/echo", "the origin sees it")
+    t.contains(t.http(url, proxy=at, method="POST", body="x=1"), "method=POST",
+               "a POST is recognised as HTTP too")
+
+    # --- as a SOCKS proxy --------------------------------------------------
+    r = t.socks_http(at, url)
+    t.eq(200, r.status, "the same port serves SOCKS5")
+    t.contains(r, "path=/echo", "the origin sees the SOCKS request")
+    t.eq(200, t.socks_http(at, url, socks4=True).status,
+         "and SOCKS4 on the same port")
+
+    # --- as a name-directed TLS proxy --------------------------------------
+    if certs and "Unknown command" not in server.output():
+        r = t.https(f"https://sni.test:{port}/echo", ca=certs.ca, strict=False,
+                    connect_to=("127.0.0.1", port))
+        t.eq(200, r.status, "and a TLS handshake, routed by the name it carries")
+        t.contains(r, "path=/echo", "which reaches the TLS origin")
+    else:
+        t.skip("auto over TLS (no SSL support, or no openssl to make certificates)")
+
+    # --- what it is not ----------------------------------------------------
+    t.not_contains(t.raw(port, "GIBBERISH\r\n\r\n"), "200 OK",
+                   "nonsense is not served as anything")

tests/cases/dnspr.py

@@ -0,0 +1,40 @@
+"""dnspr: a caching DNS proxy, answering from what it has been told."""
+
+
+def run(t):
+    port = t.free_port()
+    t.start("dnspr", f"""
+        log
+        flush
+        nserver 127.0.0.1
+        nscache 1024
+        nsrecord host.test 10.11.12.13
+        nsrecord other.test 10.11.12.14
+        nsrecord blocked.test 0.0.0.0
+        auth iponly
+        allow *
+        dnspr -p{port}
+    """)
+    # wait for the service: a datagram sent too early is simply lost
+    for _ in range(100):
+        if t.dns_query(port, "host.test"):
+            break
+
+    t.eq(["10.11.12.13"], t.dns_query(port, "host.test"),
+         "a static record is answered")
+    t.eq(["10.11.12.14"], t.dns_query(port, "other.test"),
+         "and so is another one")
+
+    # asking twice must give the same answer, which is what the cache is for
+    t.eq(["10.11.12.13"], t.dns_query(port, "host.test"),
+         "the same name answers the same way again")
+
+    # 0.0.0.0 is the documented way to make a name never resolve: the
+    # address is handed out, and it is the client that then gets nowhere
+    t.eq(["0.0.0.0"], t.dns_query(port, "blocked.test"),
+         "a name pointed at 0.0.0.0 answers with that address")
+
+    # a name it knows nothing about cannot be answered from here: the
+    # configured server does not exist, so there is nothing to forward to
+    t.ne(["10.11.12.13"], t.dns_query(port, "unknown.test") or [],
+         "an unknown name does not borrow another answer")

tests/cases/ipv6.py

@@ -0,0 +1,224 @@
+"""IPv6: listening on it, reaching it, and the rules that mention it.
+
+A service resolves IPv4 only unless told otherwise, so the proxies that are
+meant to reach IPv6 carry a family flag. Names resolving to IPv6 need
+nscache6: nscache holds the IPv4 side and nothing else.
+"""
+
+
+def run(t):
+    if not t.has_ipv6():
+        t.skip("IPv6 (this machine has no IPv6 loopback)")
+        return
+
+    origin = t.free_port()
+    v6proxy = t.free_port()
+    mixed = t.free_port()
+    v4only = t.free_port()
+    socks6 = t.free_port()
+
+    t.start("ipv6", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        http * /data data
+        httpsrv -p{origin} -i::1
+
+        # reached over IPv6, and allowed to reach IPv6
+        flush
+        auth iponly
+        allow *
+        proxy -p{v6proxy} -i::1 -6
+
+        # reached over IPv4, still able to reach IPv6
+        flush
+        auth iponly
+        allow *
+        proxy -p{mixed} -6
+
+        # asked for IPv4 only, so an IPv6 destination is not for it
+        flush
+        auth iponly
+        allow *
+        proxy -p{v4only} -4
+
+        flush
+        auth iponly
+        allow *
+        socks -p{socks6} -6
+    """, ports=[("::1", origin), ("::1", v6proxy), mixed, v4only, socks6])
+
+    url = f"http://[::1]:{origin}/echo"
+
+    # --- listening on IPv6 -------------------------------------------------
+    r = t.http(url)
+    t.eq(200, r.status, "a service bound to ::1 answers over IPv6")
+    t.contains(r, "peer.addr=::1", "the client is seen as an IPv6 address")
+    t.contains(r, "path=/echo", "and the request arrives intact")
+
+    # the Host header carries the address in brackets, and a rule matching
+    # any host still matches it
+    t.contains(r, "host=[::1]", "the host header keeps its brackets")
+
+    # --- proxying over IPv6 -------------------------------------------------
+    r = t.http(url, proxy=f"[::1]:{v6proxy}")
+    t.eq(200, r.status, "a proxy reached over IPv6 serves an IPv6 destination")
+    t.contains(r, "peer.addr=::1", "the proxy connects from IPv6 as well")
+
+    t.eq(20000, t.http(f"http://[::1]:{origin}/data?size=20000",
+                       proxy=f"[::1]:{v6proxy}").length,
+         "a body passes over IPv6")
+
+    t.eq(200, t.http(url, proxy=f"[::1]:{v6proxy}", tunnel=True).status,
+         "CONNECT works over IPv6")
+
+    # --- across the two families --------------------------------------------
+    r = t.http(url, proxy=f"127.0.0.1:{mixed}")
+    t.eq(200, r.status, "a client on IPv4 can be given an IPv6 destination")
+    t.contains(r, "peer.addr=::1", "and the far side is still reached over IPv6")
+
+    # a service told to use one family stays in it
+    t.ne(200, t.http(url, proxy=f"127.0.0.1:{v4only}").status,
+         "a service asked for IPv4 only refuses an IPv6 destination")
+
+    # --- SOCKS with an IPv6 destination -------------------------------------
+    r = t.socks_http(f"127.0.0.1:{socks6}", url)
+    t.eq(200, r.status, "SOCKS5 carries an IPv6 destination address")
+    t.contains(r, "peer.addr=::1", "which is reached over IPv6")
+
+    # --- which family a service will use --------------------------------------
+    # -46 and -64 both reach either family; -4 and -6 are each restricted to
+    # one; and nothing said means -46.
+    v4origin = t.free_port()
+    flags = {"nothing said": "", "-4": "-4", "-6": "-6", "-46": "-46", "-64": "-64"}
+    family_ports = {name: t.free_port() for name in flags}
+    sections = [f"""
+        flush
+        auth iponly
+        allow *
+        proxy -p{family_ports[name]} {flag}""" for name, flag in flags.items()]
+    t.start("ipv6_family", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        httpsrv -p{v4origin}
+        {"".join(sections)}
+    """, ports=[v4origin] + list(family_ports.values()))
+
+    expected = {
+        "nothing said": (200, None),      # -4 is the default
+        "-4": (200, None),
+        "-6": (None, 200),
+        "-46": (200, 200),
+        "-64": (200, 200),
+    }
+    for name, port in family_ports.items():
+        want4, want6 = expected[name]
+        got4 = t.http(f"http://127.0.0.1:{v4origin}/echo", proxy=f"127.0.0.1:{port}").status
+        got6 = t.http(url, proxy=f"127.0.0.1:{port}").status
+        if want4 == 200:
+            t.eq(200, got4, f"{name}: an IPv4 destination is reached")
+        else:
+            t.ne(200, got4, f"{name}: an IPv4 destination is refused")
+        if want6 == 200:
+            t.eq(200, got6, f"{name}: an IPv6 destination is reached")
+        else:
+            t.ne(200, got6, f"{name}: an IPv6 destination is refused")
+
+    # --- a name that resolves to an IPv6 address ------------------------------
+    # The two caches are separate, and the record is only kept in the one
+    # that matches the address family.
+    # separate processes: the caches belong to the process, not the service,
+    # so one section configuring nscache6 would answer for the other too
+    with_cache6 = t.free_port()
+    without = t.free_port()
+    t.start("ipv6_names", f"""
+        log
+        flush
+        nserver 127.0.0.1
+        nscache6 1024
+        nsrecord v6.test ::1
+        auth iponly
+        allow *
+        proxy -p{with_cache6} -6
+    """, ports=[with_cache6])
+    t.start("ipv6_names_nocache", f"""
+        log
+        flush
+        nserver 127.0.0.1
+        nsrecord v6.test ::1
+        auth iponly
+        allow *
+        proxy -p{without} -6
+    """, ports=[without])
+
+    t.eq(200, t.http(f"http://v6.test:{origin}/echo",
+                     proxy=f"127.0.0.1:{with_cache6}").status,
+         "a name kept in nscache6 resolves to its IPv6 address")
+    t.ne(200, t.http(f"http://v6.test:{origin}/echo",
+                     proxy=f"127.0.0.1:{without}").status,
+         "the same record without nscache6 is not there to be found")
+
+    # --- an address has more than one spelling --------------------------------
+    # Denying the IPv4 form does not deny the same host asked for as an
+    # IPv4-mapped address, nor the IPv6 loopback, which is why the security
+    # notes say to deny all of them. Both halves are checked so a change in
+    # either direction is noticed.
+    partial = t.free_port()
+    complete = t.free_port()
+    t.start("ipv6_deny", f"""
+        log
+        flush
+        auth iponly
+        deny * * 127.0.0.1
+        allow *
+        proxy -p{partial} -46
+
+        flush
+        auth iponly
+        deny * * 127.0.0.1
+        deny * * ::1
+        deny * * ::ffff:127.0.0.1
+        allow *
+        proxy -p{complete} -46
+    """, ports=[partial, complete])
+
+    v4url = f"http://127.0.0.1:{v4origin}/echo"
+    mapped = f"http://[::ffff:127.0.0.1]:{v4origin}/echo"
+
+    t.ne(200, t.http(v4url, proxy=f"127.0.0.1:{partial}").status,
+         "denying 127.0.0.1 denies the address as written")
+    t.eq(200, t.http(mapped, proxy=f"127.0.0.1:{partial}").status,
+         "but the same host asked for as ::ffff:127.0.0.1 is still reached")
+    t.eq(200, t.http(url, proxy=f"127.0.0.1:{partial}").status,
+         "and so is ::1, which the rule never mentioned")
+
+    t.ne(200, t.http(v4url, proxy=f"127.0.0.1:{complete}").status,
+         "naming every spelling denies the plain address")
+    t.ne(200, t.http(mapped, proxy=f"127.0.0.1:{complete}").status,
+         "and the mapped one")
+    t.ne(200, t.http(url, proxy=f"127.0.0.1:{complete}").status,
+         "and the IPv6 loopback")
+
+    # --- rules that name addresses ------------------------------------------
+    allowed = t.free_port()
+    refused = t.free_port()
+    t.start("ipv6_rules", f"""
+        log
+        flush
+        auth iponly
+        allow * ::1
+        proxy -p{allowed} -i::1 -6
+
+        flush
+        auth iponly
+        allow * 127.0.0.1
+        proxy -p{refused} -i::1 -6
+    """, ports=[("::1", allowed), ("::1", refused)])
+
+    t.eq(200, t.http(url, proxy=f"[::1]:{allowed}").status,
+         "a rule naming ::1 admits an IPv6 client")
+    t.ne(200, t.http(url, proxy=f"[::1]:{refused}").status,
+         "a rule naming only an IPv4 address does not")

tests/cases/parent_ports.py

@@ -26,8 +26,11 @@ def _windows():
 
 (LOW, HIGH), (ILOW, IHIGH) = _windows()
 
-# below the Linux window on purpose: the kernel ignores such a range
+# Privileged ports: the kernel ignores such a range on Linux, since it is
-UNHONOURED = (21400, 21449)
+# outside net.ipv4.ip_local_port_range, and binding them fails outright
+# without privileges. Either way nothing in the range can be taken, which
+# is the case the fallback exists for.
+UNHONOURED = (1, 99)
 
 
 def run(t):
@@ -131,6 +134,12 @@ def run(t):
     t.in_range(t.socks_udp_associate(udps), ILOW, IHIGH,
                "UDP ASSOCIATE binds inside the internal range")
 
+    # and the association still carries traffic while bound in the range
+    echo = t.udp_echo()
+    reply, bound = t.socks_udp(f"127.0.0.1:{udps}", "127.0.0.1", echo, b"data")
+    t.eq(b"echo:data", reply, "a range-bound association still relays")
+    t.in_range(bound, ILOW, IHIGH, "and the port it relays from is in the range")
+
     # without a range the association still works, on an ephemeral port
     udps2 = t.free_port()
     t.start("parent_intport_none", f"""

tests/cases/pcre.py

@@ -0,0 +1,174 @@
+"""PCRE filtering: matching, rewriting, options and rule scope.
+
+A request rewrite is applied to the buffer the server is sent, so it works
+on a direct connection as well as through a parent. The destination was
+chosen, and the access rules applied to it, before the filter ran, so a
+rewrite that moves the request to another host or changes the method is
+ignored rather than acted on.
+"""
+
+
+def _has_pcre(t):
+    """Whether this build accepts the pcre commands at all.
+
+    The last line is nonsense on purpose: it makes 3proxy report and exit
+    instead of waiting, and what it says about the line above is the answer.
+    """
+    out = t.run_config("pcre_probe",
+                       'log\npcre request deny "x"\nnot_a_command\n')
+    return "'pcre'" not in out
+
+
+def run(t):
+    if not _has_pcre(t):
+        t.skip("PCRE (this build has no PCRE support)")
+        return
+
+    origin = t.free_port()
+    t.start("pcre_origin", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        http * /secret* echo
+        http * /data data
+        httpsrv -p{origin}
+    """, ports=[origin])
+
+    url = f"http://127.0.0.1:{origin}"
+
+    def proxy_with(name, *rules):
+        port = t.free_port()
+        t.start(name, "\n".join([
+            "log", "flush", "auth iponly", "allow *", *rules, f"proxy -p{port}"]),
+            ports=[port])
+        return f"127.0.0.1:{port}"
+
+    # --- matching and denial ---------------------------------------------
+    p = proxy_with("deny", 'pcre request deny "/secret"')
+    t.eq(200, t.http(url + "/echo", proxy=p).status, "an unmatched request passes")
+    t.ne(200, t.http(url + "/secret/page", proxy=p).status, "a matched request is denied")
+
+    # the rules are ordered, and the first decision wins
+    p = proxy_with("allow_first", 'pcre request allow "/echo"', 'pcre request deny "/"')
+    t.eq(200, t.http(url + "/echo", proxy=p).status, "allow short-circuits a later deny")
+    p = proxy_with("deny_first", 'pcre request deny "/"', 'pcre request allow "/echo"')
+    t.ne(200, t.http(url + "/echo", proxy=p).status, "deny short-circuits a later allow")
+
+    # --- what the pattern is matched against ------------------------------
+    p = proxy_with("cliheader", 'pcre cliheader deny "BadBot"')
+    t.eq(200, t.http(url + "/echo", proxy=p).status, "a header rule ignores other requests")
+    t.ne(200, t.http(url + "/echo", proxy=p, headers={"User-Agent": "BadBot/1.0"}).status,
+         "a client header can be matched")
+
+    # --- options ------------------------------------------------------------
+    p = proxy_with("caseless", "pcre_options PCRE2_CASELESS",
+                   'pcre request deny "/SECRET"')
+    t.ne(200, t.http(url + "/secret/page", proxy=p).status,
+         "PCRE2_CASELESS makes the match case-insensitive")
+    p = proxy_with("cased", 'pcre request deny "/SECRET"')
+    t.eq(200, t.http(url + "/secret/page", proxy=p).status,
+         "without it the match is case-sensitive")
+
+    # --- the access rule a pcre rule carries --------------------------------
+    p = proxy_with("ace_here", f'pcre request deny "/echo" * * * {origin}')
+    t.ne(200, t.http(url + "/echo", proxy=p).status,
+         "a rule applies where its access rule matches")
+    p = proxy_with("ace_elsewhere", 'pcre request deny "/echo" * * * 1')
+    t.eq(200, t.http(url + "/echo", proxy=p).status,
+         "and not where it does not")
+
+    # pcre_extend appends another access rule to the one just defined
+    p = proxy_with("extend", 'pcre request deny "/echo" * * * 1',
+                   f"pcre_extend * * * {origin}")
+    t.ne(200, t.http(url + "/echo", proxy=p).status,
+         "pcre_extend widens the rule to another destination")
+    p = proxy_with("extend_other", 'pcre request deny "/echo" * * * 1',
+                   "pcre_extend * * * 2")
+    t.eq(200, t.http(url + "/echo", proxy=p).status,
+         "an extension that matches nothing changes nothing")
+
+    # --- rewriting the reply ------------------------------------------------
+    p = proxy_with("rewrite_srv",
+                   'pcre_rewrite srvheader dunno "text/plain" "text/rewritten"',
+                   'pcre_rewrite srvdata dunno "peer.addr" "PEER.ADDR"')
+    r = t.http(url + "/echo", proxy=p)
+    t.eq(200, r.status, "a rewritten reply still arrives")
+    t.eq("text/rewritten", r.header("Content-Type"), "a reply header can be rewritten")
+    t.contains(r, "PEER.ADDR", "reply data can be rewritten")
+    t.not_contains(r, "peer.addr", "the original text is gone")
+
+    # --- rewriting the request ------------------------------------------------
+    p = proxy_with("rewrite_req", 'pcre_rewrite request dunno "/echo/old" "/echo/new"')
+    r = t.http(url + "/echo/old", proxy=p)
+    t.eq(200, r.status, "a rewritten request still arrives")
+    t.contains(r, "path=/echo/new", "the origin sees the rewritten path")
+
+    # the replacement may be longer or shorter than what it replaces
+    p = proxy_with("rewrite_long", 'pcre_rewrite request dunno "/echo/x" "/echo/deeper/still"')
+    t.contains(t.http(url + "/echo/x", proxy=p), "path=/echo/deeper/still",
+               "a longer replacement is spliced in")
+    p = proxy_with("rewrite_short", 'pcre_rewrite request dunno "/echo/aaaaaaaaaa" "/echo/b"')
+    t.contains(t.http(url + "/echo/aaaaaaaaaa", proxy=p), "path=/echo/b",
+               "a shorter replacement is spliced in")
+
+    p = proxy_with("rewrite_query", 'pcre_rewrite request dunno "token=old" "token=new"')
+    t.contains(t.http(url + "/echo?token=old", proxy=p), "query=token=new",
+               "the query can be rewritten")
+
+    p = proxy_with("rewrite_none", 'pcre_rewrite request dunno "/nothing" "/else"')
+    t.contains(t.http(url + "/echo/keep", proxy=p), "path=/echo/keep",
+               "a request that does not match is left alone")
+
+    # what follows the request line has to survive the splice
+    p = proxy_with("rewrite_post", 'pcre_rewrite request dunno "/echo/old" "/echo/new"')
+    r = t.http(url + "/echo/old", proxy=p, method="POST", body="hello",
+               headers={"Content-Type": "text/plain"})
+    t.contains(r, "path=/echo/new", "a POST is rewritten too")
+    t.contains(r, "content.length=5", "its body is still described correctly")
+
+    conn = t.connection("127.0.0.1", origin, proxy=p)
+    try:
+        first = t.http(url + "/echo/old", proxy=p, conn=conn)
+        second = t.http(url + "/echo/old", proxy=p, conn=conn)
+        t.contains(first, "path=/echo/new", "the first of two on a connection is rewritten")
+        t.contains(second, "path=/echo/new", "and so is the second")
+    finally:
+        conn.close()
+
+    # --- rewrites that would change where the request goes --------------------
+    elsewhere = t.free_port()
+    t.start("pcre_elsewhere", f"""
+        log
+        flush
+        auth iponly
+        allow *
+        http * /echo* echo
+        httpsrv -p{elsewhere}
+    """, ports=[elsewhere])
+
+    p = proxy_with("rewrite_host",
+                   f'pcre_rewrite request dunno "127.0.0.1:{origin}" "127.0.0.1:{elsewhere}"')
+    r = t.http(url + "/echo", proxy=p)
+    t.eq(200, r.status, "a rewrite naming another host still answers")
+    t.contains(r, f"host=127.0.0.1:{origin}",
+               "but the request goes where the access rules allowed")
+
+    p = proxy_with("rewrite_method", 'pcre_rewrite request dunno "^GET" "HEAD"')
+    t.contains(t.http(url + "/echo", proxy=p), "method=GET",
+               "a rewrite of the method is ignored")
+
+    # --- and the same rewrite through an HTTP parent --------------------------
+    parent = t.free_port()
+    t.start("pcre_parent", f"""
+        log
+        flush
+        auth iponly
+        allow *
+        proxy -p{parent}
+    """, ports=[parent])
+    p = proxy_with("rewrite_parent", 'pcre_rewrite request dunno "/echo/old" "/echo/new"',
+                   f"parent 1000 http 127.0.0.1 {parent}")
+    r = t.http(url + "/echo/old", proxy=p)
+    t.eq(200, r.status, "a rewritten request through a parent arrives")
+    t.contains(r, "path=/echo/new", "the origin sees the rewritten path through a parent")

tests/cases/portmap.py

@@ -0,0 +1,64 @@
+"""The port mappers: tcppm forwards a TCP port, udppm a UDP one."""
+
+
+def run(t):
+    # --- tcppm ---------------------------------------------------------
+    origin = t.free_port()
+    mapped = t.free_port()
+    refused = t.free_port()
+
+    t.start("portmap_tcp", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        http * /data data
+        httpsrv -p{origin}
+
+        flush
+        auth iponly
+        allow *
+        tcppm {mapped} 127.0.0.1 {origin}
+
+        flush
+        auth iponly
+        deny *
+        tcppm {refused} 127.0.0.1 {origin}
+    """, ports=[origin, mapped, refused])
+
+    r = t.http(f"http://127.0.0.1:{mapped}/echo")
+    t.eq(200, r.status, "a mapped TCP port reaches the target")
+    t.contains(r, "path=/echo", "the target sees the request")
+    t.contains(r, "peer.addr=127.0.0.1", "the mapper makes the connection")
+
+    t.eq(20000, t.http(f"http://127.0.0.1:{mapped}/data?size=20000").length,
+         "a body passes through the mapper")
+
+    # the mapper is a service like any other, so its rules apply
+    r = t.http(f"http://127.0.0.1:{refused}/echo")
+    t.ne(200, r.status, "a mapper whose rules deny the client answers nothing")
+
+    t.stop_all()
+
+    # --- udppm ---------------------------------------------------------
+    # something has to be listening for the mapped datagrams to go anywhere
+    echo = t.udp_echo()
+    mapped = t.free_port()
+    t.start("portmap_udp", f"""
+        log
+        flush
+        auth iponly
+        allow *
+        udppm {mapped} 127.0.0.1 {echo}
+    """)
+    # a UDP service has no listening socket to wait for, so ask until it
+    # answers rather than racing it
+    t.wait_udp(mapped)
+    t.eq(b"echo:hello", t.udp_exchange(mapped, b"hello"),
+         "a datagram is relayed and the reply comes back")
+    t.eq(b"echo:second", t.udp_exchange(mapped, b"second"),
+         "a second datagram uses the mapping again")
+
+    big = b"x" * 2000
+    t.eq(b"echo:" + big, t.udp_exchange(mapped, big),
+         "a larger datagram survives the round trip")

tests/cases/socks.py

@@ -46,6 +46,22 @@ def run(t):
     t.eq(200, t.socks_http(plain, origin + "/echo", socks4=True).status,
          "a SOCKS4 connection")
 
+    # --- the UDP association, and what goes through it ---------------------
+    # Binding the association is one thing; carrying a datagram is what it
+    # is for.
+    echo = t.udp_echo()
+    reply, bound = t.socks_udp(plain, "127.0.0.1", echo, b"ping")
+    t.eq(b"echo:ping", reply, "a datagram is relayed and answered")
+    t.ne(None, bound, "the association reports the port to send to")
+
+    reply, _ = t.socks_udp(plain, "127.0.0.1", echo, b"x" * 2000)
+    t.eq(b"echo:" + b"x" * 2000, reply, "a larger datagram survives the relay")
+
+    # each association gets its own socket
+    _, first = t.socks_udp(plain, "127.0.0.1", echo, b"one")
+    _, second = t.socks_udp(plain, "127.0.0.1", echo, b"two")
+    t.ne(first, second, "a second association binds its own port")
+
     # --- authentication ----------------------------------------------------
     t.eq(200, t.socks_http(guarded, origin + "/echo",
                            auth=("alice", "secret")).status,

tests/cases/ssl.py

@@ -0,0 +1,203 @@
+"""TLS: a proxy wrapped in TLS, one chained to another over TLS, and MITM.
+
+The key material is generated for the run, so nothing long-lived lives in
+the tree. Cases skip when the build has no TLS or openssl is missing.
+"""
+
+
+def _no_tls(t, server):
+    """True when the binary rejected the TLS commands in a configuration."""
+    return "Unknown command" in server
+
+
+def run(t):
+    certs = t.certs()
+    if not certs:
+        t.skip("TLS (openssl is not available to generate certificates)")
+        return
+
+    # The key material has to be sound before anything is asked of the
+    # proxy, or every failure below points at the wrong thing.
+    if not certs.verified:
+        t.fail("the generated certificate chain verifies", "OK",
+               certs.verify_output or "openssl verify failed")
+        return
+    t.ok("the generated certificate chain verifies")
+
+    # --- a proxy wrapped in TLS (ssl_serv) ----------------------------
+    origin = t.free_port()
+    tlsproxy = t.free_port()
+
+    server = t.start("ssl_serv", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        httpsrv -p{origin}
+
+        flush
+        ssl_server_cert {certs.server}
+        ssl_server_key {certs.server_key}
+        ssl_serv
+        auth iponly
+        allow *
+        proxy -p{tlsproxy}
+    """, ports=[origin, tlsproxy])
+
+    if _no_tls(t, server.output()):
+        t.skip("TLS (this build has no SSL support)")
+        return
+
+    url = f"http://127.0.0.1:{origin}/echo"
+    r = t.tls_proxy_http(f"127.0.0.1:{tlsproxy}", url, ca=certs.ca)
+    t.eq(200, r.status, "a proxy wrapped in TLS serves a request")
+    t.contains(r, "path=/echo", "the origin sees the request made over TLS")
+
+    # a client holding a different CA must not accept the certificate
+    bad = t.tls_proxy_http(f"127.0.0.1:{tlsproxy}", url, ca=certs.other)
+    t.ne(200, bad.status, "a client that does not trust the CA is refused")
+    t.contains(bad, "CERTIFICATE_VERIFY_FAILED",
+               "the refusal is a certificate verification failure")
+
+    # and plain HTTP must not get through a TLS listener
+    t.ne(200, t.http(url, proxy=f"127.0.0.1:{tlsproxy}").status,
+         "a plain request to the TLS port is refused")
+
+    t.stop_all()
+
+    # --- a TLS client chained to a TLS server -------------------------
+    # The ssl_serv proxy is the parent; the ssl_cli proxy reaches it over
+    # TLS and verifies it against the CA.
+    origin = t.free_port()
+    parent = t.free_port()
+    client = t.free_port()
+
+    server = t.start("ssl_chain", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        http * /data data
+        httpsrv -p{origin}
+
+        flush
+        ssl_server_cert {certs.server}
+        ssl_server_key {certs.server_key}
+        ssl_serv
+        auth iponly
+        allow *
+        proxy -p{parent}
+
+        flush
+        ssl_noserv
+        auth iponly
+        allow *
+        parent 1000 connects 127.0.0.1 {parent}
+        ssl_client_mode 3
+        ssl_client_ca_file {certs.ca}
+        ssl_client_verify
+        ssl_cli
+        proxy -p{client}
+    """, ports=[origin, parent, client])
+
+    through = f"127.0.0.1:{client}"
+    r = t.http(f"http://127.0.0.1:{origin}/echo", proxy=through)
+    t.eq(200, r.status, "a request through the TLS chain arrives")
+    t.contains(r, "path=/echo", "the origin sees the chained request")
+
+    # the origin is reached by the parent, not by the client proxy
+    t.contains(r, "peer.addr=127.0.0.1", "the parent makes the final connection")
+
+    t.eq(10000, t.http(f"http://127.0.0.1:{origin}/data?size=10000",
+                       proxy=through).length,
+         "a body survives the TLS chain")
+    t.eq(10000, t.http(f"http://127.0.0.1:{origin}/data?size=10000&chunked=1",
+                       proxy=through).length,
+         "a chunked body survives the TLS chain")
+
+    t.stop_all()
+
+    # --- MITM ----------------------------------------------------------
+    # The origin runs in its own process so the proxy log holds only what
+    # the proxy saw, and an https origin gives the tunnel something real to
+    # carry.
+    origin = t.free_port()
+    t.start("ssl_mitm_origin", f"""
+        log
+        ssl_server_cert {certs.server}
+        ssl_server_key {certs.server_key}
+        ssl_serv
+        auth iponly
+        allow *
+        http * /secret* echo
+        httpsrv -p{origin}
+    """, ports=[origin])
+
+    mitm = t.free_port()
+    plain = t.free_port()
+    proxies = t.start("ssl_mitm", f"""
+        log
+        nserver 127.0.0.1
+        nscache 1024
+        nsrecord intercepted.test 127.0.0.1
+        ssl_server_ca_file {certs.ca}
+        ssl_server_ca_key {certs.ca_key}
+        ssl_certcache {certs.cache}
+        ssl_client_ca_file {certs.ca}
+        ssl_mitm
+        auth iponly
+        allow *
+        proxy -p{mitm}
+
+        flush
+        ssl_nomitm
+        ssl_nocli
+        auth iponly
+        allow *
+        proxy -p{plain}
+    """, ports=[mitm, plain])
+
+    # A name the proxy resolves itself through nsrecord, so the request
+    # carries a hostname the way a real one would, without depending on
+    # what the machine running the tests puts in its hosts file.
+    target = f"https://intercepted.test:{origin}/secret/page"
+
+    # The client trusts our CA, which is what signs the spoofed certificate,
+    # and checks it the way a current client does. The certificate names the
+    # upstream host rather than the one asked for, so the chain is verified
+    # but the name is not.
+    r = t.https(target, proxy=f"127.0.0.1:{mitm}", ca=certs.ca,
+                verify_name=False)
+    if r.status is None and "Authority Key Identifier" in (r.error or ""):
+        # A build against wolfSSL cannot generate certificate extensions,
+        # so the identifiers a strict verifier looks for are absent there.
+        t.skip("strict verification of an intercepted certificate "
+               "(this build cannot generate the key identifiers)")
+        r = t.https(target, proxy=f"127.0.0.1:{mitm}", ca=certs.ca,
+                    strict=False, verify_name=False)
+    else:
+        t.ok("the intercepted certificate satisfies a strict verifier")
+    t.eq(200, r.status, "MITM passes the request through")
+    t.contains(r, "path=/secret/page", "the intercepted request reaches the origin")
+
+    # the point of interception: the decrypted request line reaches the log
+    log = t.wait_output(proxies, "/secret/page")
+    t.contains(log, "/secret/page", "MITM puts the request URI in the log")
+    t.contains(log, "GET", "MITM logs the method")
+    t.contains(log, "intercepted.test", "MITM logs the host that was asked for")
+
+    # a client that does not trust the CA sees the substitution
+    refused = t.https(target, proxy=f"127.0.0.1:{mitm}", ca=certs.other,
+                      strict=False, verify_name=False)
+    t.ne(200, refused.status, "MITM is visible to a client with another CA")
+
+    # Without interception the same request is opaque: the proxy logs the
+    # CONNECT target and nothing from inside the tunnel.
+    before = len(proxies.output())
+    r = t.https(target, proxy=f"127.0.0.1:{plain}", ca=certs.ca,
+                verify_name=False)
+    t.eq(200, r.status, "the plain proxy tunnels the same request")
+    tunnelled = t.wait_output(proxies, "intercepted.test", since=before)
+    t.contains(tunnelled, "intercepted.test", "the tunnel logs the CONNECT target")
+    t.not_contains(tunnelled, "/secret/page",
+                   "a tunnelled request keeps its URI out of the log")

tests/cases/tlspr.py

@@ -0,0 +1,47 @@
+"""tlspr: the destination comes from the name in the TLS handshake."""
+
+
+def run(t):
+    certs = t.certs()
+    if not certs:
+        t.skip("tlspr (openssl is not available to generate certificates)")
+        return
+
+    origin = t.free_port()
+    sni = t.free_port()
+
+    server = t.start("tlspr", f"""
+        log
+        ssl_server_cert {certs.server}
+        ssl_server_key {certs.server_key}
+        ssl_serv
+        auth iponly
+        allow *
+        http * /echo* echo
+        httpsrv -p{origin}
+
+        flush
+        ssl_noserv
+        nserver 127.0.0.1
+        nscache 1024
+        nsrecord sni.test 127.0.0.1
+        auth iponly
+        allow *
+        tlspr -p{sni} -P{origin}
+    """, ports=[origin, sni])
+
+    if "Unknown command" in server.output():
+        t.skip("tlspr (this build has no SSL support)")
+        return
+
+    # The certificate names sni.test, so the name in the handshake is both
+    # what picks the destination and what the client checks.
+    r = t.https(f"https://sni.test:{sni}/echo", ca=certs.ca, strict=False,
+                connect_to=("127.0.0.1", sni))
+    t.eq(200, r.status, "the name in the handshake reaches its destination")
+    t.contains(r, "path=/echo", "the request arrives at the origin")
+
+    # a name the proxy cannot resolve has nowhere to go
+    r = t.https(f"https://nowhere.test:{sni}/echo", ca=certs.ca, strict=False,
+                verify_name=False, connect_to=("127.0.0.1", sni))
+    t.ne(200, r.status, "a name that does not resolve is refused")

tests/harness.py

@@ -22,11 +22,14 @@ configurations it needs, starts them, and states what it expects:
 import base64
 import http.client
 import os
+import shutil
 import socket
+import ssl
 import struct
 import subprocess
 import sys
 import textwrap
+import threading
 import time
 
 
@@ -85,6 +88,27 @@ class Server:
                 self.proc.wait(timeout=5)
 
 
+class Certs:
+    """A test CA, a certificate it signed, and somewhere to cache spoofed ones.
+
+    Paths use forward slashes: they are written into configurations read by
+    3proxy, and ssl_certcache insists on a trailing separator.
+    """
+
+    def __init__(self, directory):
+        self.dir = directory.replace("\\", "/")
+        self.ca = self.dir + "/ca.pem"
+        self.ca_key = self.dir + "/ca.key"
+        self.server = self.dir + "/server.pem"
+        self.server_key = self.dir + "/server.key"
+        # a second CA nothing is signed by, for the cases that must fail
+        self.other = self.dir + "/other.pem"
+        self.other_key = self.dir + "/other.key"
+        self.cache = self.dir + "/cache/"
+        self.verified = False
+        self.verify_output = ""
+
+
 class Failure(Exception):
     """Raised when a case cannot go on, e.g. a server refused to start."""
 
@@ -100,9 +124,26 @@ class Tester:
         self.checks = []
         self.timeout = 10
         self._skipped = 0
+        self._certs = None
+        self.logs = []
+        self.udp_servers = []
 
     # ---- servers -----------------------------------------------------
 
+    def has_ipv6(self):
+        """Whether this machine can use the IPv6 loopback at all."""
+        try:
+            sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        except OSError:
+            return False
+        try:
+            sock.bind(("::1", 0))
+            return True
+        except OSError:
+            return False
+        finally:
+            sock.close()
+
     def free_port(self):
         """A port nothing is listening on. Closed again before it is used,
         which is racy in principle and reliable enough in practice."""
@@ -123,7 +164,11 @@ class Tester:
         return path
 
     def start(self, name, config, ports=()):
-        """Write a configuration, run it, and wait for its ports to open."""
+        """Write a configuration, run it, and wait for its ports to open.
+
+        A port may be given as a number, or as (address, port) for a service
+        bound somewhere other than 127.0.0.1.
+        """
         path = self.write_config(name, config)
         logfile = os.path.join(self.tmpdir, name + ".out")
         with open(logfile, "wb") as out:
@@ -132,8 +177,9 @@ class Tester:
         server = Server(name, path, proc, logfile)
         self.servers.append(server)
 
-        for port in ports:
+        for entry in ports:
-            if not self.wait_port(port):
+            host, port = entry if isinstance(entry, tuple) else ("127.0.0.1", entry)
+            if not self.wait_port(port, host=host):
                 code = proc.poll()
                 if code is None:
                     died = "the process is still running"
@@ -154,19 +200,38 @@ class Tester:
                               stderr=subprocess.STDOUT, timeout=15)
         return done.stdout.decode("utf-8", "replace")
 
-    def wait_port(self, port, timeout=5.0):
+    def wait_port(self, port, timeout=5.0, host="127.0.0.1"):
         deadline = time.time() + timeout
         while time.time() < deadline:
             try:
-                with socket.create_connection(("127.0.0.1", port), 0.25):
+                with socket.create_connection((host, port), 0.25):
                     return True
             except OSError:
                 time.sleep(0.02)
         return False
 
+    def wait_output(self, server, needle, timeout=5.0, since=0):
+        """Wait for a server to log something.
+
+        A record is written when the connection it describes finishes, not
+        when the reply reaches the client, so reading straight after a
+        request usually finds nothing yet.
+        """
+        deadline = time.time() + timeout
+        while True:
+            text = server.output()[since:]
+            if needle in text or time.time() > deadline:
+                return text
+            time.sleep(0.05)
+
     def stop_all(self):
+        """Stop the servers, keeping what they printed for the report."""
+        for sock in self.udp_servers:
+            sock.close()
+        self.udp_servers = []
         for server in self.servers:
             server.stop()
+            self.logs.append((server.name, server.output()))
         self.servers = []
 
     # ---- requests ----------------------------------------------------
@@ -196,7 +261,10 @@ class Tester:
                                        tunnel=tunnel)
             target = path
             if proxy and not tunnel:
-                target = f"http://{host}:{port}{path}"
+                # an address with colons goes back in brackets, or the
+                # absolute URI cannot be read
+                authority = f"[{host}]" if ":" in host else host
+                target = f"http://{authority}:{port}{path}"
             if body is not None and not isinstance(body, bytes):
                 body = body.encode()
             conn.request(method, target, body=body, headers=headers)
@@ -253,6 +321,150 @@ class Tester:
         except OSError as exc:
             return f"<no reply: {exc}>"
 
+    # ---- UDP ---------------------------------------------------------
+
+    def udp_echo(self, prefix=b"echo:"):
+        """Start a UDP server that echoes what it receives, and give its port.
+
+        Something has to be on the far side of a port mapper or a SOCKS
+        association for the data path to be visible at all.
+        """
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.bind(("127.0.0.1", 0))
+        port = sock.getsockname()[1]
+
+        def serve():
+            while True:
+                try:
+                    data, peer = sock.recvfrom(65536)
+                except OSError:
+                    return
+                try:
+                    sock.sendto(prefix + data, peer)
+                except OSError:
+                    return
+
+        thread = threading.Thread(target=serve, daemon=True)
+        thread.start()
+        self.udp_servers.append(sock)
+        return port
+
+    def udp_exchange(self, port, payload, host="127.0.0.1"):
+        """Send one datagram and return the reply, or None."""
+        if not isinstance(payload, bytes):
+            payload = payload.encode()
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.settimeout(self.timeout)
+        try:
+            sock.sendto(payload, (host, port))
+            return sock.recvfrom(65536)[0]
+        except OSError:
+            return None
+        finally:
+            sock.close()
+
+    def wait_udp(self, port, payload=b"ping", timeout=5.0):
+        """Wait until a UDP service answers.
+
+        There is no socket to connect to, so readiness can only be found
+        out by asking; a datagram sent before the service is up is simply
+        lost.
+        """
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            if self.udp_exchange(port, payload) is not None:
+                return True
+            time.sleep(0.05)
+        return False
+
+    def socks_udp(self, socks, host, port, payload, keep=None):
+        """Relay a datagram through a SOCKS5 association.
+
+        Returns (reply payload, association port), or (None, port) if
+        nothing came back. The control connection has to stay open for the
+        association to live, so it is closed only on the way out.
+        """
+        if not isinstance(payload, bytes):
+            payload = payload.encode()
+        shost, sport = self._hostport(socks)
+        ctrl = None
+        udp = None
+        try:
+            ctrl = socket.create_connection((shost, sport), self.timeout)
+            ctrl.settimeout(self.timeout)
+            ctrl.sendall(b"\x05\x01\x00")
+            if self._recvall(ctrl, 2) != b"\x05\x00":
+                return None, None
+            ctrl.sendall(b"\x05\x03\x00\x01\x00\x00\x00\x00" + struct.pack("!H", 0))
+            reply = self._recvall(ctrl, 4)
+            if len(reply) < 4 or reply[1] != 0:
+                return None, None
+            _, bound = self._read_socks_addr(ctrl, reply[3])
+
+            udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            udp.settimeout(self.timeout)
+            header = (b"\x00\x00\x00\x01" + socket.inet_aton(host) +
+                      struct.pack("!H", port))
+            udp.sendto(header + payload, (shost, bound))
+            try:
+                data = udp.recvfrom(65536)[0]
+            except OSError:
+                return None, bound
+            # the reply carries the same kind of header, which is not payload
+            if len(data) < 10 or data[3] != 1:
+                return None, bound
+            return data[10:], bound
+        except OSError:
+            return None, None
+        finally:
+            if udp:
+                udp.close()
+            if ctrl:
+                ctrl.close()
+
+    # ---- DNS ---------------------------------------------------------
+
+    def dns_query(self, port, name, host="127.0.0.1"):
+        """Ask for an A record and return the addresses in the answer."""
+        query = struct.pack("!HHHHHH", 0x2A2A, 0x0100, 1, 0, 0, 0)
+        for label in name.split("."):
+            query += bytes([len(label)]) + label.encode()
+        query += b"\x00" + struct.pack("!HH", 1, 1)
+
+        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        sock.settimeout(self.timeout)
+        try:
+            sock.sendto(query, (host, port))
+            data = sock.recvfrom(65536)[0]
+        except OSError:
+            return None
+        finally:
+            sock.close()
+
+        if len(data) < 12 or data[:2] != query[:2]:
+            return None
+        answers = struct.unpack("!H", data[6:8])[0]
+        addresses = []
+        pos = 12
+        while pos < len(data) and data[pos]:      # skip the question
+            pos += data[pos] + 1
+        pos += 5
+        for _ in range(answers):
+            if pos + 12 > len(data):
+                break
+            if data[pos] & 0xC0 == 0xC0:
+                pos += 2
+            else:
+                while pos < len(data) and data[pos]:
+                    pos += data[pos] + 1
+                pos += 1
+            rtype, _, _, rdlen = struct.unpack("!HHIH", data[pos:pos + 10])
+            pos += 10
+            if rtype == 1 and rdlen == 4:
+                addresses.append(socket.inet_ntoa(data[pos:pos + 4]))
+            pos += rdlen
+        return addresses
+
     # ---- SOCKS -------------------------------------------------------
 
     def _socks_connect(self, shost, sport, host, port, socks4=False,
@@ -289,6 +501,8 @@ class Tester:
 
             if remote_dns:
                 target = b"\x03" + bytes([len(host)]) + host.encode()
+            elif ":" in host:
+                target = b"\x04" + socket.inet_pton(socket.AF_INET6, host)
             else:
                 target = b"\x01" + socket.inet_aton(socket.gethostbyname(host))
             sock.sendall(b"\x05\x01\x00" + target + struct.pack("!H", port))
@@ -378,6 +592,169 @@ class Tester:
             data += piece
         return data
 
+    # ---- TLS ---------------------------------------------------------
+
+    def certs(self):
+        """A CA and a certificate for 127.0.0.1, generated once per run.
+
+        Returns None when openssl is unavailable, so a case can skip rather
+        than fail on a machine that cannot make key material.
+        """
+        if self._certs is not None:
+            return self._certs or None
+        if not shutil.which("openssl"):
+            self._certs = False
+            return None
+
+        c = Certs(os.path.join(self.tmpdir, "certs"))
+        os.makedirs(c.cache, exist_ok=True)
+        csr = c.dir + "/server.csr"
+        ext = c.dir + "/server.ext"
+        ca_ext = c.dir + "/ca.ext"
+        # The key identifiers are spelled out because LibreSSL does not add
+        # them for a signed certificate the way OpenSSL 3 does, and Python
+        # rejects a chain with no Authority Key Identifier from 3.13.
+        with open(ext, "w") as fp:
+            fp.write("subjectAltName=IP:127.0.0.1,DNS:localhost,DNS:sni.test\n"
+                     "subjectKeyIdentifier=hash\n"
+                     "authorityKeyIdentifier=keyid,issuer\n")
+        # A CA without these is not usable as one. They go in a file rather
+        # than in -addext, which LibreSSL - the openssl on a stock macOS -
+        # does not apply the same way.
+        with open(ca_ext, "w") as fp:
+            fp.write("basicConstraints=critical,CA:TRUE\n"
+                     "keyUsage=critical,keyCertSign,cRLSign\n"
+                     "subjectKeyIdentifier=hash\n")
+
+        def ca_steps(key, csr_path, out, name):
+            return [
+                ["openssl", "genrsa", "-out", key, "2048"],
+                ["openssl", "req", "-new", "-nodes", "-key", key,
+                 "-subj", "/CN=" + name, "-out", csr_path],
+                ["openssl", "x509", "-req", "-in", csr_path, "-signkey", key,
+                 "-days", "3650", "-sha256", "-extfile", ca_ext, "-out", out],
+            ]
+
+        steps = (
+            ca_steps(c.ca_key, c.dir + "/ca.csr", c.ca, "3proxy-test-ca") +
+            ca_steps(c.other_key, c.dir + "/other.csr", c.other,
+                     "3proxy-test-other-ca") +
+            [
+                ["openssl", "genrsa", "-out", c.server_key, "2048"],
+                ["openssl", "req", "-new", "-key", c.server_key,
+                 "-subj", "/CN=127.0.0.1", "-out", csr],
+                ["openssl", "x509", "-req", "-in", csr, "-CA", c.ca,
+                 "-CAkey", c.ca_key, "-CAcreateserial", "-out", c.server,
+                 "-days", "3650", "-sha256", "-extfile", ext],
+            ])
+        for step in steps:
+            done = subprocess.run(step, stdout=subprocess.PIPE,
+                                  stderr=subprocess.STDOUT, timeout=60)
+            if done.returncode:
+                self._certs = False
+                return None
+
+        # If the chain does not verify, the fault is in the generation, not
+        # in whatever is about to present it.
+        # -x509_strict is what a current client applies, so check that here
+        # rather than discovering it in a handshake.
+        check = subprocess.run(["openssl", "verify", "-x509_strict",
+                                "-CAfile", c.ca, c.server],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.STDOUT, timeout=60)
+        c.verified = check.returncode == 0
+        c.verify_output = check.stdout.decode("utf-8", "replace").strip()
+
+        self._certs = c
+        return c
+
+    def _context(self, ca=None, strict=True, verify_name=True):
+        """A client context.
+
+        strict=False drops the RFC 5280 checks Python turns on by default
+        from 3.13, which reject a certificate with no Authority Key
+        Identifier. verify_name=False keeps the chain check but ignores
+        which host the certificate names, for the intercepted connections
+        where that is the upstream identity rather than the one asked for.
+        """
+        if ca:
+            context = ssl.create_default_context(cafile=ca)
+            if not strict:
+                context.verify_flags &= ~getattr(ssl, "VERIFY_X509_STRICT", 0)
+            if not verify_name:
+                context.check_hostname = False
+            return context
+        context = ssl.create_default_context()
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+        return context
+
+    def tls_proxy_http(self, proxy, url, ca=None, strict=True, method="GET",
+                       body=None, headers=None):
+        """A request to a proxy that is itself wrapped in TLS (ssl_serv)."""
+        host, port, path = self._split(url)
+        phost, pport = self._hostport(proxy)
+        try:
+            raw = socket.create_connection((phost, pport), self.timeout)
+            sock = self._context(ca, strict).wrap_socket(raw, server_hostname=phost)
+        except (OSError, ssl.SSLError) as exc:
+            return Response(error=f"{type(exc).__name__}: {exc}")
+
+        conn = http.client.HTTPConnection(host, port, timeout=self.timeout)
+        conn.sock = sock
+        try:
+            if body is not None and not isinstance(body, bytes):
+                body = body.encode()
+            authority = f"[{host}]" if ":" in host else host
+            conn.request(method, f"http://{authority}:{port}{path}", body=body,
+                         headers=headers or {})
+            reply = conn.getresponse()
+            return Response(reply.status, reply.read(), dict(reply.getheaders()))
+        except (OSError, http.client.HTTPException) as exc:
+            return Response(error=f"{type(exc).__name__}: {exc}")
+        finally:
+            conn.close()
+
+    def https(self, url, proxy=None, ca=None, strict=True, verify_name=True,
+              method="GET", headers=None, connect_to=None):
+        """An https:// request, optionally tunnelled through a proxy.
+
+        connect_to sends the handshake somewhere other than the name in the
+        URL, which is how a name-directed proxy is reached: the name still
+        goes out in the handshake and is what the certificate is checked
+        against.
+        """
+        host, port, path = self._split(url, default_port=443)
+        context = self._context(ca, strict, verify_name)
+        try:
+            if connect_to:
+                raw = socket.create_connection(connect_to, self.timeout)
+                conn = http.client.HTTPSConnection(host, port, context=context,
+                                                   timeout=self.timeout)
+                conn.sock = context.wrap_socket(raw, server_hostname=host)
+                conn.request(method, path, headers=headers or {})
+                reply = conn.getresponse()
+                return Response(reply.status, reply.read(),
+                                dict(reply.getheaders()))
+            if proxy:
+                phost, pport = self._hostport(proxy)
+                conn = http.client.HTTPSConnection(phost, pport, context=context,
+                                                   timeout=self.timeout)
+                conn.set_tunnel(host, port)
+            else:
+                conn = http.client.HTTPSConnection(host, port, context=context,
+                                                   timeout=self.timeout)
+            conn.request(method, path, headers=headers or {})
+            reply = conn.getresponse()
+            return Response(reply.status, reply.read(), dict(reply.getheaders()))
+        except (OSError, ssl.SSLError, http.client.HTTPException) as exc:
+            return Response(error=f"{type(exc).__name__}: {exc}")
+        finally:
+            try:
+                conn.close()
+            except (OSError, NameError, UnboundLocalError):
+                pass
+
     # ---- helpers -----------------------------------------------------
 
     @staticmethod
@@ -388,16 +765,31 @@ class Tester:
 
     @staticmethod
     def _hostport(value):
+        if value.startswith("["):
+            host, _, rest = value[1:].partition("]")
+            return host, int(rest[1:])
         host, _, port = value.rpartition(":")
         return host or "127.0.0.1", int(port)
 
     @staticmethod
-    def _split(url):
+    def _split(url, default_port=80):
-        prefix = "http://"
+        """Split a URL, understanding an address in brackets.
-        if url.startswith(prefix):
+
-            url = url[len(prefix):]
+        The brackets are dropped: they belong to the URL, not to the address
+        a socket call or a certificate check wants.
+        """
+        for prefix in ("http://", "https://"):
+            if url.startswith(prefix):
+                url = url[len(prefix):]
+                break
         authority, _, path = url.partition("/")
-        host, _, port = authority.rpartition(":")
+        if authority.startswith("["):
+            host, _, rest = authority[1:].partition("]")
+            port = rest[1:] if rest.startswith(":") else default_port
+        elif ":" in authority:
+            host, _, port = authority.rpartition(":")
+        else:
+            host, port = authority, default_port
         return host or "127.0.0.1", int(port), "/" + path
 
     # ---- assertions --------------------------------------------------

tests/run.py

@@ -113,6 +113,15 @@ def main():
                     if actual is not None:
                         print(f"         actual:   {actual}")
 
+            if tester.checks and any(status is False for status, _, _, _ in tester.checks):
+                for name, text in tester.logs:
+                    lines = [line for line in text.splitlines() if line.strip()]
+                    if not lines:
+                        continue
+                    print(f"    --- {name} said ---")
+                    for line in lines[-12:]:
+                        print(f"      {line}")
+
             if error:
                 failed += 1
                 failures.append(f"{name}: case aborted")