Add TLS tests: a wrapped proxy, a TLS chain, and MITM

Three arrangements, with key material generated for the run rather than
kept in the tree: a proxy wrapped in TLS, a proxy that reaches a TLS parent
and verifies it against the CA, and MITM.

The MITM case checks what interception is for: the decrypted request line,
URI and all, reaches the log, where the same request through a plain
CONNECT tunnel leaves only the host and port.

The origin runs in its own process there so the proxy log holds only what
the proxy saw, and log assertions wait, since a record is written when the
connection finishes rather than when the reply arrives.

Verification of the spoofed certificate is deliberately not strict: 3proxy
issues those without an Authority Key Identifier, which Python rejects
under its 3.13 defaults.

tests/README.md

@@ -7,7 +7,9 @@
     python3 tests/run.py --keep              # keep the configurations and logs
 
 Python 3.6 or later and a built 3proxy are the only requirements: the suite
-is standard library throughout, so it runs wherever 3proxy builds. With no
+is standard library throughout, so it runs wherever 3proxy builds. The TLS
+case additionally wants `openssl` on PATH to generate its key material, and
+skips itself when that is missing or the build has no TLS support. With no
 `--bin` it looks in `bin/`, then `build/bin/`, then the per-configuration
 directories a multi-configuration CMake generator uses.
 
@@ -49,6 +51,13 @@ Assertions are `eq`, `ne`, `contains`, `not_contains`, `in_range`,
 `not_in_range`, plus `ok`, `fail` and `skip`. `harness.field()` and
 `int_field()` pull a single line out of an `echo` reply.
 
+`t.certs()` generates a CA, a second unrelated CA, and a certificate for
+127.0.0.1, once per run and inside the run's temporary directory, so no key
+material lives in the tree. `t.https()`, `t.tls_proxy_http()` and
+`t.socks_http()` reach a server through TLS, a TLS-wrapped proxy, or SOCKS.
+Log records are written when a connection finishes rather than when the
+reply arrives, so assert on them through `t.wait_output(server, text)`.
+
 Note that access rules accumulate until `flush`, so a service section that
 means to stand on its own should start with one - otherwise an earlier
 `allow *` matches first and the rule under test is never reached.

tests/cases/ssl.py

@@ -0,0 +1,177 @@
+"""TLS: a proxy wrapped in TLS, one chained to another over TLS, and MITM.
+
+The key material is generated for the run, so nothing long-lived lives in
+the tree. Cases skip when the build has no TLS or openssl is missing.
+"""
+
+
+def _no_tls(t, server):
+    """True when the binary rejected the TLS commands in a configuration."""
+    return "Unknown command" in server
+
+
+def run(t):
+    certs = t.certs()
+    if not certs:
+        t.skip("TLS (openssl is not available to generate certificates)")
+        return
+
+    # --- a proxy wrapped in TLS (ssl_serv) ----------------------------
+    origin = t.free_port()
+    tlsproxy = t.free_port()
+
+    server = t.start("ssl_serv", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        httpsrv -p{origin}
+
+        flush
+        ssl_server_cert {certs.server}
+        ssl_server_key {certs.server_key}
+        ssl_serv
+        auth iponly
+        allow *
+        proxy -p{tlsproxy}
+    """, ports=[origin, tlsproxy])
+
+    if _no_tls(t, server.output()):
+        t.skip("TLS (this build has no SSL support)")
+        return
+
+    url = f"http://127.0.0.1:{origin}/echo"
+    r = t.tls_proxy_http(f"127.0.0.1:{tlsproxy}", url, ca=certs.ca)
+    t.eq(200, r.status, "a proxy wrapped in TLS serves a request")
+    t.contains(r, "path=/echo", "the origin sees the request made over TLS")
+
+    # a client holding a different CA must not accept the certificate
+    bad = t.tls_proxy_http(f"127.0.0.1:{tlsproxy}", url, ca=certs.other)
+    t.ne(200, bad.status, "a client that does not trust the CA is refused")
+    t.contains(bad, "CERTIFICATE_VERIFY_FAILED",
+               "the refusal is a certificate verification failure")
+
+    # and plain HTTP must not get through a TLS listener
+    t.ne(200, t.http(url, proxy=f"127.0.0.1:{tlsproxy}").status,
+         "a plain request to the TLS port is refused")
+
+    t.stop_all()
+
+    # --- a TLS client chained to a TLS server -------------------------
+    # The ssl_serv proxy is the parent; the ssl_cli proxy reaches it over
+    # TLS and verifies it against the CA.
+    origin = t.free_port()
+    parent = t.free_port()
+    client = t.free_port()
+
+    server = t.start("ssl_chain", f"""
+        log
+        auth iponly
+        allow *
+        http * /echo* echo
+        http * /data data
+        httpsrv -p{origin}
+
+        flush
+        ssl_server_cert {certs.server}
+        ssl_server_key {certs.server_key}
+        ssl_serv
+        auth iponly
+        allow *
+        proxy -p{parent}
+
+        flush
+        ssl_noserv
+        auth iponly
+        allow *
+        parent 1000 connects 127.0.0.1 {parent}
+        ssl_client_mode 3
+        ssl_client_ca_file {certs.ca}
+        ssl_client_verify
+        ssl_cli
+        proxy -p{client}
+    """, ports=[origin, parent, client])
+
+    through = f"127.0.0.1:{client}"
+    r = t.http(f"http://127.0.0.1:{origin}/echo", proxy=through)
+    t.eq(200, r.status, "a request through the TLS chain arrives")
+    t.contains(r, "path=/echo", "the origin sees the chained request")
+
+    # the origin is reached by the parent, not by the client proxy
+    t.contains(r, "peer.addr=127.0.0.1", "the parent makes the final connection")
+
+    t.eq(10000, t.http(f"http://127.0.0.1:{origin}/data?size=10000",
+                       proxy=through).length,
+         "a body survives the TLS chain")
+    t.eq(10000, t.http(f"http://127.0.0.1:{origin}/data?size=10000&chunked=1",
+                       proxy=through).length,
+         "a chunked body survives the TLS chain")
+
+    t.stop_all()
+
+    # --- MITM ----------------------------------------------------------
+    # The origin runs in its own process so the proxy log holds only what
+    # the proxy saw, and an https origin gives the tunnel something real to
+    # carry.
+    origin = t.free_port()
+    t.start("ssl_mitm_origin", f"""
+        log
+        ssl_server_cert {certs.server}
+        ssl_server_key {certs.server_key}
+        ssl_serv
+        auth iponly
+        allow *
+        http * /secret* echo
+        httpsrv -p{origin}
+    """, ports=[origin])
+
+    mitm = t.free_port()
+    plain = t.free_port()
+    proxies = t.start("ssl_mitm", f"""
+        log
+        ssl_server_ca_file {certs.ca}
+        ssl_server_ca_key {certs.ca_key}
+        ssl_certcache {certs.cache}
+        ssl_client_ca_file {certs.ca}
+        ssl_mitm
+        auth iponly
+        allow *
+        proxy -p{mitm}
+
+        flush
+        ssl_nomitm
+        ssl_nocli
+        auth iponly
+        allow *
+        proxy -p{plain}
+    """, ports=[mitm, plain])
+
+    target = f"https://127.0.0.1:{origin}/secret/page"
+
+    # The client trusts our CA, which is what signs the spoofed certificate.
+    # Verification is not strict: 3proxy issues those certificates without an
+    # Authority Key Identifier, which Python rejects under its 3.13 defaults.
+    r = t.https(target, proxy=f"127.0.0.1:{mitm}", ca=certs.ca, strict=False)
+    t.eq(200, r.status, "MITM passes the request through")
+    t.contains(r, "path=/secret/page", "the intercepted request reaches the origin")
+
+    # the point of interception: the decrypted request line reaches the log
+    log = t.wait_output(proxies, "/secret/page")
+    t.contains(log, "/secret/page", "MITM puts the request URI in the log")
+    t.contains(log, "GET", "MITM logs the method")
+    t.contains(log, str(origin), "MITM logs the destination")
+
+    # a client that does not trust the CA sees the substitution
+    refused = t.https(target, proxy=f"127.0.0.1:{mitm}", ca=certs.other,
+                      strict=False)
+    t.ne(200, refused.status, "MITM is visible to a client with another CA")
+
+    # Without interception the same request is opaque: the proxy logs the
+    # CONNECT target and nothing from inside the tunnel.
+    before = len(proxies.output())
+    r = t.https(target, proxy=f"127.0.0.1:{plain}", ca=certs.ca, strict=False)
+    t.eq(200, r.status, "the plain proxy tunnels the same request")
+    tunnelled = t.wait_output(proxies, str(origin), since=before)
+    t.contains(tunnelled, str(origin), "the tunnel logs the CONNECT target")
+    t.not_contains(tunnelled, "/secret/page",
+                   "a tunnelled request keeps its URI out of the log")

tests/harness.py

@@ -22,7 +22,9 @@ configurations it needs, starts them, and states what it expects:
 import base64
 import http.client
 import os
+import shutil
 import socket
+import ssl
 import struct
 import subprocess
 import sys
@@ -85,6 +87,25 @@ class Server:
                 self.proc.wait(timeout=5)
 
 
+class Certs:
+    """A test CA, a certificate it signed, and somewhere to cache spoofed ones.
+
+    Paths use forward slashes: they are written into configurations read by
+    3proxy, and ssl_certcache insists on a trailing separator.
+    """
+
+    def __init__(self, directory):
+        self.dir = directory.replace("\\", "/")
+        self.ca = self.dir + "/ca.pem"
+        self.ca_key = self.dir + "/ca.key"
+        self.server = self.dir + "/server.pem"
+        self.server_key = self.dir + "/server.key"
+        # a second CA nothing is signed by, for the cases that must fail
+        self.other = self.dir + "/other.pem"
+        self.other_key = self.dir + "/other.key"
+        self.cache = self.dir + "/cache/"
+
+
 class Failure(Exception):
     """Raised when a case cannot go on, e.g. a server refused to start."""
 
@@ -100,6 +121,7 @@ class Tester:
         self.checks = []
         self.timeout = 10
         self._skipped = 0
+        self._certs = None
 
     # ---- servers -----------------------------------------------------
 
@@ -164,6 +186,20 @@ class Tester:
                 time.sleep(0.02)
         return False
 
+    def wait_output(self, server, needle, timeout=5.0, since=0):
+        """Wait for a server to log something.
+
+        A record is written when the connection it describes finishes, not
+        when the reply reaches the client, so reading straight after a
+        request usually finds nothing yet.
+        """
+        deadline = time.time() + timeout
+        while True:
+            text = server.output()[since:]
+            if needle in text or time.time() > deadline:
+                return text
+            time.sleep(0.05)
+
     def stop_all(self):
         for server in self.servers:
             server.stop()
@@ -378,6 +414,120 @@ class Tester:
             data += piece
         return data
 
+    # ---- TLS ---------------------------------------------------------
+
+    def certs(self):
+        """A CA and a certificate for 127.0.0.1, generated once per run.
+
+        Returns None when openssl is unavailable, so a case can skip rather
+        than fail on a machine that cannot make key material.
+        """
+        if self._certs is not None:
+            return self._certs or None
+        if not shutil.which("openssl"):
+            self._certs = False
+            return None
+
+        c = Certs(os.path.join(self.tmpdir, "certs"))
+        os.makedirs(c.cache, exist_ok=True)
+        csr = c.dir + "/server.csr"
+        ext = c.dir + "/server.ext"
+        with open(ext, "w") as fp:
+            fp.write("subjectAltName=IP:127.0.0.1,DNS:localhost\n")
+
+        # OpenSSL 3 refuses to trust a CA without these extensions
+        ca_ext = ["-addext", "basicConstraints=critical,CA:TRUE",
+                  "-addext", "keyUsage=critical,keyCertSign,cRLSign"]
+        steps = [
+            ["openssl", "genrsa", "-out", c.ca_key, "2048"],
+            ["openssl", "req", "-x509", "-new", "-nodes", "-key", c.ca_key,
+             "-sha256", "-days", "3650", "-subj", "/CN=3proxy-test-ca",
+             "-out", c.ca] + ca_ext,
+            ["openssl", "genrsa", "-out", c.other_key, "2048"],
+            ["openssl", "req", "-x509", "-new", "-nodes", "-key", c.other_key,
+             "-sha256", "-days", "3650", "-subj", "/CN=3proxy-test-other-ca",
+             "-out", c.other] + ca_ext,
+            ["openssl", "genrsa", "-out", c.server_key, "2048"],
+            ["openssl", "req", "-new", "-key", c.server_key,
+             "-subj", "/CN=127.0.0.1", "-out", csr],
+            ["openssl", "x509", "-req", "-in", csr, "-CA", c.ca,
+             "-CAkey", c.ca_key, "-CAcreateserial", "-out", c.server,
+             "-days", "3650", "-sha256", "-extfile", ext],
+        ]
+        for step in steps:
+            done = subprocess.run(step, stdout=subprocess.PIPE,
+                                  stderr=subprocess.STDOUT, timeout=60)
+            if done.returncode:
+                self._certs = False
+                return None
+
+        self._certs = c
+        return c
+
+    def _context(self, ca=None, strict=True):
+        """A client context. strict=False drops the RFC 5280 checks Python
+        turns on by default from 3.13, which reject a certificate with no
+        Authority Key Identifier."""
+        if ca:
+            context = ssl.create_default_context(cafile=ca)
+            if not strict:
+                context.verify_flags &= ~getattr(ssl, "VERIFY_X509_STRICT", 0)
+            return context
+        context = ssl.create_default_context()
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+        return context
+
+    def tls_proxy_http(self, proxy, url, ca=None, strict=True, method="GET",
+                       body=None, headers=None):
+        """A request to a proxy that is itself wrapped in TLS (ssl_serv)."""
+        host, port, path = self._split(url)
+        phost, pport = self._hostport(proxy)
+        try:
+            raw = socket.create_connection((phost, pport), self.timeout)
+            sock = self._context(ca, strict).wrap_socket(raw, server_hostname=phost)
+        except (OSError, ssl.SSLError) as exc:
+            return Response(error=f"{type(exc).__name__}: {exc}")
+
+        conn = http.client.HTTPConnection(host, port, timeout=self.timeout)
+        conn.sock = sock
+        try:
+            if body is not None and not isinstance(body, bytes):
+                body = body.encode()
+            conn.request(method, f"http://{host}:{port}{path}", body=body,
+                         headers=headers or {})
+            reply = conn.getresponse()
+            return Response(reply.status, reply.read(), dict(reply.getheaders()))
+        except (OSError, http.client.HTTPException) as exc:
+            return Response(error=f"{type(exc).__name__}: {exc}")
+        finally:
+            conn.close()
+
+    def https(self, url, proxy=None, ca=None, strict=True, method="GET",
+              headers=None):
+        """An https:// request, optionally tunnelled through a proxy."""
+        host, port, path = self._split(url, default_port=443)
+        context = self._context(ca, strict)
+        try:
+            if proxy:
+                phost, pport = self._hostport(proxy)
+                conn = http.client.HTTPSConnection(phost, pport, context=context,
+                                                   timeout=self.timeout)
+                conn.set_tunnel(host, port)
+            else:
+                conn = http.client.HTTPSConnection(host, port, context=context,
+                                                   timeout=self.timeout)
+            conn.request(method, path, headers=headers or {})
+            reply = conn.getresponse()
+            return Response(reply.status, reply.read(), dict(reply.getheaders()))
+        except (OSError, ssl.SSLError, http.client.HTTPException) as exc:
+            return Response(error=f"{type(exc).__name__}: {exc}")
+        finally:
+            try:
+                conn.close()
+            except (OSError, NameError, UnboundLocalError):
+                pass
+
     # ---- helpers -----------------------------------------------------
 
     @staticmethod
@@ -392,12 +542,16 @@ class Tester:
         return host or "127.0.0.1", int(port)
 
     @staticmethod
-    def _split(url):
+    def _split(url, default_port=80):
-        prefix = "http://"
+        for prefix in ("http://", "https://"):
             if url.startswith(prefix):
                 url = url[len(prefix):]
+                break
         authority, _, path = url.partition("/")
+        if ":" in authority:
             host, _, port = authority.rpartition(":")
+        else:
+            host, port = authority, default_port
         return host or "127.0.0.1", int(port), "/" + path
 
     # ---- assertions --------------------------------------------------