Fix rewrite and over-long header handling on Windows

op_rewrite checked what it built with targetunsafe(), which describes a
path on this machine: on Windows it must name a drive or a share, so a
rewritten request path was refused and every rewrite rule failed there.
A rewrite produces a request path and is checked as one.

The header loop stopped at HTTPSRV_MAXHDR and answered anyway, leaving
the rest of the request in the stream for the next one to be read out
of. It refuses the request instead. A header longer than the buffer
arrives as several lines, so the count bounds what is read rather than
what a client may send in one header.

The test for the buffer growth after a PCRE rewrite sent its request
through to an httpsrv origin, which stops reading at that same cap; it
uses an origin which reads whatever it is sent, since what is under test
is the proxy in the middle.

CMakeLists.txt

@@ -169,7 +169,7 @@ if(WIN32)
     endif()
 
     # Windows libraries
-    set(WINDOWS_LIBS ws2_32 advapi32 user32 kernel32 gdi32 crypt32)
+    set(WINDOWS_LIBS ws2_32 mswsock advapi32 user32 kernel32 gdi32 crypt32)
 
     # Windows plugins (always built)
     set(DEFAULT_PLUGINS

Makefile.msvc

@@ -24,7 +24,7 @@ LN = link
 LDFLAGS =  /nologo /subsystem:console /incremental:no
 DLFLAGS = /DLL
 DLSUFFICS = .dll
-LIBS = ws2_32.lib advapi32.lib odbc32.lib user32.lib kernel32.lib Gdi32.lib Crypt32.lib $(SSL_LIBS) pcre2-8.lib
+LIBS = ws2_32.lib mswsock.lib advapi32.lib odbc32.lib user32.lib kernel32.lib Gdi32.lib Crypt32.lib $(SSL_LIBS) pcre2-8.lib
 LIBSPREFIX =
 LIBSSUFFIX = .lib
 LIBEXT = .lib

Makefile.watcom

@@ -14,7 +14,7 @@ LN = link
 LDFLAGS = /nologo /subsystem:console /incremental:no 
 DLFLAGS = /DLL
 DLSUFFICS = .dll
-LIBS = ws2_32.lib advapi32.lib user32.lib kernel32.lib
+LIBS = ws2_32.lib mswsock.lib advapi32.lib user32.lib kernel32.lib
 LIBSPREFIX = 
 LIBSSUFFIX = .lib
 LIBEXT = .lib                                                                                               

Makefile.win

@@ -30,7 +30,7 @@ ifeq ($(HTTPSRV),true)
 endif
 DLFLAGS ?= -shared
 DLSUFFICS = .dll
-LIBS += -lws2_32 -lodbc32 -ladvapi32 -luser32 -lbcrypt
+LIBS += -lws2_32 -lmswsock -lodbc32 -ladvapi32 -luser32 -lbcrypt
 LIBSPREFIX = -l
 LIBSSUFFIX =
 LNOUT = -o 

doc/html/howtoe.html

@@ -34,6 +34,7 @@
  <li><a href="#ISFTP">How to set up an FTP proxy</a></li>
  <li><a href="#TLSPR">How to set up an SNI proxy (tlspr)</a></li>
  <li><a href="#DNSPR">How to set up a DNS proxy (dnspr)</a></li>
+ <li><a href="#HTTPSRV">How to serve pages with the built-in HTTP server (httpsrv)</a></li>
  <li><a href="#SSLPLUGIN">How to set up TLS/SSL (https proxy, mTLS)</a></li>
  <li><a href="#CERTIFICATES">How to create CA and certificates for SSL</a></li>
  <li><a href="#PCRE">How to use PCRE filtering (regular expressions)</a></li>
@@ -727,6 +728,218 @@ nscache 65536
 nscache6 65536
 dnspr -p53 -F10.0.0.1
 </pre>
+ </p>
+ <li><a name="HTTPSRV"><i>How to serve pages with the built-in HTTP server (httpsrv)</i></a>
+<p>
+httpsrv answers requests itself instead of forwarding them. What it does with a
+request is decided by <code>http</code> rules written before the service, the way
+access rules are: the first rule whose host and URL both match handles the
+request. It is useful for a status page, a small static site, a block page for
+requests an ACL rejects, or a health check an upstream balancer can poll.
+</p><pre>
+http OPERATION HOST URL [PARAMETERS]
+</pre>
+<p>
+HOST is matched against the Host header, URL against the path with the query
+string removed. A minimal static site:
+</p><pre>
+auth iponly
+allow *
+
+http file * /           /usr/local/web/index.html
+http file * /*.html     "/usr/local/web/$1.html"
+http file * /css/*.css  "/usr/local/web/css/$1.css"
+http cache * /img/**    "/usr/local/web/img/$1" * 3600
+httpsrv -p80 -i127.0.0.1
+</pre>
+<p>
+<b>Patterns.</b> <code>*</code> stands for any run of characters within one
+element of the path and does not cross a <code>/</code>, so a rule cannot reach
+into a directory it did not name. <code>**</code> crosses them. Each star, and
+each group of a regular expression, is remembered in order: <code>$1</code>
+upwards stand for them in the path or location the rule builds, and
+<code>$0</code> for the whole request path. A <code>rewrite_host</code> rule
+uses the stars of its own host pattern instead, since that is what it is
+rewriting. With a PCRE build a pattern may be
+written as a regular expression with a <code>pcre:</code> prefix, for the URL and
+for the host alike:
+</p><pre>
+http file  * /d/*.txt       "/usr/local/web/$1.txt"
+http cache * "pcre:^/(.*)/pic/(.*)\.(gif|jpeg)$" "/usr/local/web/picts/$1/$2.$3"
+http file  status.example.com /**  "/usr/local/web/status/$1"
+http file  "pcre:^(www|web)\.example\.com$" /**  "/usr/local/web/$1"
+</pre>
+<p>
+Outside quotes a dollar begins the name of a file to include, so an argument
+holding one - a path built with <code>$1</code>, a regular expression anchored
+with <code>$</code> - is written in quotes, as above. <code>$$</code> stands for
+a single dollar and is not read as an include either.
+</p>
+<p>
+<b>Operations.</b>
+</p><pre>
+&#35; file - send the file, using sendfile/TransmitFile where the system can
+http file * /dl/**  "/usr/local/web/dl/$1"
+
+&#35; cache - read it into memory on the first request and answer from there
+http cache * /css/*.css "/usr/local/web/css/$1.css"
+
+&#35; redir - answer with a redirect, 302 unless a status is given
+http redir * /old/**  301 "https://example.org/$1"
+
+&#35; rewrite - change the path and hand the request to the rules after this one
+http rewrite * /alias/** "/w/$1"
+
+&#35; rewrite_host - the same for the host, which decides which rules match next
+http rewrite_host *.old.example ** "$1.new.example"
+
+&#35; reply - a status and nothing else
+http reply * /health**  200 "X-Health: ok"
+http reply * /down**    503 "Retry-After: 30"
+
+&#35; echo, data - describe the request, or generate content of a given size
+http echo * /echo**
+http data * /gen** size=1048576
+</pre>
+<p>
+<b>What a rule adds to the answer.</b> <code>file</code> and <code>cache</code>
+take, after the path, a content type, a max-age, headers to add and a status to
+answer with. Each may be left out or written as <code>*</code>:
+</p><pre>
+http OPERATION HOST URL PATH [TYPE [MAX-AGE [HEADERS [CODE]]]]
+
+&#35; type worked out from the name, cached by clients for an hour
+http file * /img/*.png "/usr/local/web/img/$1.png" * 3600
+
+&#35; a type of its own, and a header
+http file * /api/*.json "/usr/local/web/api/$1.json" application/json * "X-Api: 1"
+
+&#35; a file serving as the body of an error page
+http file * /err/** /usr/local/web/404.html text/html * * 404
+</pre>
+<p>
+HEADERS is one argument holding whole header lines, separated by a backslash and
+an n - the two characters, since a configuration line cannot carry a line
+ending. Quote it, headers contain spaces. A rule's headers and max-age go with
+whatever status that rule asked for, but not with a refusal the server itself
+decided on: a request for a file which is not there is answered 404 by the
+server, not by the rule.
+</p>
+<p>
+Types not known to the server are registered with
+<code>http_content_type</code>, and a type named by a rule is used whatever the
+name of the file says:
+</p><pre>
+http_content_type .webp image/webp
+http_content_type wasm  application/wasm
+</pre>
+<p>
+<b>Files and dates.</b> Only a full path is taken - a relative one would be read
+against whatever directory the service happens to be in - and a path holding
+<code>.</code> or <code>..</code> as an element, a line ending or a star is
+refused. On Windows a path must name a drive or a share (<code>"C:\web\$1"</code>
+or <code>"\\host\share\$1"</code>). A request which decodes to a path leaving the
+tree is refused before any of this. Every answer carries Last-Modified, and a
+request carrying If-Modified-Since is answered 304 with no body when the file
+has not changed.
+</p>
+<p>
+<b>Caching.</b> <code>cache</code> reads the file once and answers from memory
+afterwards; a file which has changed on disk is read again, and one larger than
+a megabyte is sent as <code>file</code> would. With a MAX-AGE the file is not
+looked at again for that long - the rule has already told clients the file may
+be treated as unchanged for that time - so a request costs nothing but the copy
+out. Without one every request stats the file and a change is picked up at once.
+</p>
+<p>
+<b>A block page.</b> A service which rejects a request with a redirect can send
+the client to an httpsrv running beside it:
+</p><pre>
+auth iponly
+deny * * "pcre:^(ads|track)[0-9]*\.example\.(com|net)$"
+allow *
+proxy -p3128 -i192.168.1.1
+
+flush
+auth iponly
+allow *
+http file * /**  /usr/local/web/blocked.html text/html * * 403
+httpsrv -p8080 -i127.0.0.1
+</pre>
+<p>
+<b>Both a site and a proxy.</b> A request may arrive the way it arrives at a
+site - a path, with the name in the Host header - or the way it arrives at a
+proxy, naming the whole URL, or the host alone with CONNECT. Both are read. A
+proxy-form request authenticates with Proxy-Authorization and is refused with
+407, the way a proxy refuses one; a site-form request uses Authorization and
+401. What answers it is decided by the rules either way.
+</p>
+<p>
+<code>proxypass</code> is the rule which answers by fetching, so one service can
+serve what it has and proxy the rest:
+</p><pre>
+auth iponly
+allow *
+http file * /local/** "/usr/local/web/$1"
+http proxypass * /**
+httpsrv -p8080
+</pre>
+<p>
+An access rule redirecting to the local proxy does the same without a rule for
+it. The chain with no address is what "the local proxy" is written as, and the
+second <code>allow</code> is what the proxy matches on the pass it makes itself,
+since a rule carrying the chain is not taken twice:
+</p><pre>
+auth iponly
+allow *
+parent 1000 http 0.0.0.0 0
+allow *
+http file * /local/** "/usr/local/web/$1"
+httpsrv -p8080
+</pre>
+<p>
+Authentication happens twice for the same reason - once for the service, once
+for the proxy - so a configuration which asks for credentials asks for them the
+way a proxy does.
+</p>
+<p>
+The access rules are read from the top on both passes, and the second one is
+where the request's destination is known. On the first pass the service is
+answering for itself, so an address or a port in a rule is matched against the
+address the client connected to; the name from the request is matched on both
+passes. On the second the destination is the one the request names, so rules
+written with an address, a port or a name decide what the proxy may fetch, and
+they decide it before it connects:
+</p><pre>
+auth iponly
+allow *
+parent 1000 http 0.0.0.0 0
+allow * * * 80,443
+deny *
+httpsrv -p8080
+</pre>
+<p>
+Everything reaches the rules, only ports 80 and 443 are fetched, and a
+<code>deny</code> written before the rule carrying the chain applies on both
+passes just the same. The connection to the server is kept for the next request and
+closed when that request goes elsewhere, or when the server has closed it in the
+meantime.
+</p>
+<p>
+<b>Connections.</b> A client asking in HTTP/1.1 gets a 1.1 answer and the
+connection is kept for the next request, unless it sent
+<code>Connection: close</code>; a 1.0 client has to ask for keep-alive. The
+connection is only kept when the length of the answer is known exactly, which is
+true of every operation except the administration pages, so those are always the
+last thing on a connection. A request body the server cannot read to its end -
+one sent chunked, or one larger than a megabyte - ends the connection too.
+</p>
+<p>
+<b>Administration.</b> The <code>admin</code> service is httpsrv with the pages
+of the administration interface already declared, see
+<a href="#ADMIN">Administering and information analysis</a>. Rules may be added
+before it in the same way, and are taken first.
+</p>
  </p>
  <li><a name="SSLPLUGIN"><i>How to set up TLS/SSL (https proxy, mTLS)</i></a>
 <p>
@@ -1223,6 +1436,34 @@ pcre_extend deny * 192.168.0.1/16
 <p>
 <b>Note:</b> Regular expressions don't require authentication and cannot replace
 authentication and/or allow/deny ACLs.
+</p>
+<p>
+<b>Regular expressions in host names:</b> a host name in the target list of an
+access rule may be written as a regular expression instead of a wildmask, by
+giving it a <code>pcre:</code> prefix (<code>regex:</code> means the same). This
+needs a build with PCRE support, the same as the <code>pcre</code> commands
+above.
+</p><pre>
+&#35; Wildmask: a name may only be matched at its beginning and its end
+deny * * *ads.example.com
+
+&#35; Regular expression: anything PCRE can express
+deny * * "pcre:^(ads|track)[0-9]*\.example\.(com|net)$"
+allow * * "pcre:^(www|api)\.example\.com$"
+</pre>
+<p>
+The name is lowercased and trailing dots are removed before the pattern is
+matched, so write patterns in lower case. Quote a pattern which ends in
+<code>$</code>, or write it as <code>$$</code>: outside quotes a lone dollar
+begins the name of a file to include. Only the target list takes names - the
+source list is addresses - and the name is only checked when the request
+carries one. A wildmask is cheaper and is enough for most rules; a regular
+expression is matched per request.
+</p>
+<p>
+The same prefix and the same patterns are used by the <code>http</code> command
+of the built-in HTTP server, for the host a rule answers for and for the URL it
+matches.
 </p>
  <li><A NAME="AUTH">How to limit service access</a>
 <p>

doc/html/howtor.html

@@ -34,6 +34,7 @@
  <li><a href="#ISFTP">Как настроить FTP прокси?</a></li>
  <li><a href="#TLSPR">Как настроить SNI proxy (tlspr)</a></li>
  <li><a href="#DNSPR">Как настроить DNS proxy (dnspr)</a></li>
+ <li><a href="#HTTPSRV">Как отдавать страницы встроенным HTTP-сервером (httpsrv)</a></li>
  <li><a href="#SSLPLUGIN">Как настроить TLS/SSL (https прокси, mTLS)</a></li>
  <li><a href="#CERTIFICATES">Как создать CA и сертификаты для SSL</a></li>
  <li><a href="#PCRE">Как использовать PCRE-фильтрацию (регулярные выражения)</a></li>
@@ -738,6 +739,217 @@ dnspr -p53 -F10.0.0.1
 </pre>
  </p>
 
+ <li><a name="HTTPSRV"><i>Как отдавать страницы встроенным HTTP-сервером (httpsrv)</i></a>
+<p>
+httpsrv отвечает на запросы сам, а не пересылает их. Что делать с запросом,
+определяют правила <code>http</code>, записанные перед сервисом, как и правила
+доступа: запрос обрабатывает первое правило, у которого совпали и хост, и URL.
+Это удобно для страницы состояния, небольшого статического сайта, страницы
+блокировки для запросов, отклонённых ACL, или health check, который опрашивает
+вышестоящий балансировщик.
+</p><pre>
+http ОПЕРАЦИЯ ХОСТ URL [ПАРАМЕТРЫ]
+</pre>
+<p>
+ХОСТ сопоставляется с заголовком Host, URL - с путём без строки запроса.
+Минимальный статический сайт:
+</p><pre>
+auth iponly
+allow *
+
+http file * /           /usr/local/web/index.html
+http file * /*.html     "/usr/local/web/$1.html"
+http file * /css/*.css  "/usr/local/web/css/$1.css"
+http cache * /img/**    "/usr/local/web/img/$1" * 3600
+httpsrv -p80 -i127.0.0.1
+</pre>
+<p>
+<b>Шаблоны.</b> <code>*</code> означает любую последовательность символов внутри
+одного элемента пути и не пересекает <code>/</code>, поэтому правило не может
+попасть в каталог, который не назван в нём. <code>**</code> пересекает.
+Каждая звёздочка и каждая группа регулярного выражения запоминаются по порядку:
+<code>$1</code> и далее подставляют их в путь или адрес, который строит правило,
+<code>$0</code> - весь путь запроса. Правило <code>rewrite_host</code>
+использует звёздочки собственного шаблона хоста, поскольку переписывает именно
+его. В сборке с PCRE шаблон можно записать
+регулярным выражением с префиксом <code>pcre:</code> - и для URL, и для хоста:
+</p><pre>
+http file  * /d/*.txt       "/usr/local/web/$1.txt"
+http cache * "pcre:^/(.*)/pic/(.*)\.(gif|jpeg)$" "/usr/local/web/picts/$1/$2.$3"
+http file  status.example.com /**  "/usr/local/web/status/$1"
+http file  "pcre:^(www|web)\.example\.com$" /**  "/usr/local/web/$1"
+</pre>
+<p>
+Вне кавычек доллар начинает имя включаемого файла, поэтому аргумент, содержащий
+доллар - путь с <code>$1</code>, регулярное выражение с якорем <code>$</code> -
+записывается в кавычках, как выше. <code>$$</code> означает один доллар и тоже
+не читается как включение файла.
+</p>
+<p>
+<b>Операции.</b>
+</p><pre>
+&#35; file - отдать файл, через sendfile/TransmitFile там, где система это умеет
+http file * /dl/**  "/usr/local/web/dl/$1"
+
+&#35; cache - прочитать в память при первом запросе и отвечать из неё
+http cache * /css/*.css "/usr/local/web/css/$1.css"
+
+&#35; redir - ответить редиректом, 302, если код не задан
+http redir * /old/**  301 "https://example.org/$1"
+
+&#35; rewrite - изменить путь и передать запрос следующим правилам
+http rewrite * /alias/** "/w/$1"
+
+&#35; rewrite_host - то же для хоста, от которого зависит выбор следующих правил
+http rewrite_host *.old.example ** "$1.new.example"
+
+&#35; reply - только код ответа, без тела
+http reply * /health**  200 "X-Health: ok"
+http reply * /down**    503 "Retry-After: 30"
+
+&#35; echo, data - описание запроса или генерация содержимого заданного размера
+http echo * /echo**
+http data * /gen** size=1048576
+</pre>
+<p>
+<b>Что правило добавляет в ответ.</b> <code>file</code> и <code>cache</code>
+принимают после пути тип содержимого, max-age, добавляемые заголовки и код
+ответа. Любой из них можно опустить или записать как <code>*</code>:
+</p><pre>
+http ОПЕРАЦИЯ ХОСТ URL ПУТЬ [ТИП [MAX-AGE [ЗАГОЛОВКИ [КОД]]]]
+
+&#35; тип определяется по имени файла, клиенты кэшируют час
+http file * /img/*.png "/usr/local/web/img/$1.png" * 3600
+
+&#35; собственный тип и заголовок
+http file * /api/*.json "/usr/local/web/api/$1.json" application/json * "X-Api: 1"
+
+&#35; файл как тело страницы ошибки
+http file * /err/** /usr/local/web/404.html text/html * * 404
+</pre>
+<p>
+ЗАГОЛОВКИ - один аргумент, содержащий целые строки заголовков, разделённые
+обратной косой чертой и n - двумя символами, поскольку строка конфигурации не
+может содержать конец строки. Аргумент нужно брать в кавычки, в заголовках есть
+пробелы. Заголовки и max-age правила отправляются с тем кодом, который правило
+задало, но не с отказом, который решил вернуть сам сервер: на запрос
+отсутствующего файла 404 отвечает сервер, а не правило.
+</p>
+<p>
+Неизвестные серверу типы регистрируются командой
+<code>http_content_type</code>, а тип, названный в правиле, используется
+независимо от имени файла:
+</p><pre>
+http_content_type .webp image/webp
+http_content_type wasm  application/wasm
+</pre>
+<p>
+<b>Файлы и даты.</b> Принимается только полный путь - относительный отсчитывался
+бы от того каталога, в котором оказался сервис, - а путь с элементом
+<code>.</code> или <code>..</code>, концом строки или звёздочкой отвергается. В
+Windows путь должен указывать диск или сетевой ресурс (<code>"C:\web\$1"</code>
+или <code>"\\host\share\$1"</code>). Запрос, который декодируется в путь за
+пределами дерева, отвергается раньше всего этого. Каждый ответ содержит
+Last-Modified, а запрос с If-Modified-Since получает 304 без тела, если файл не
+изменился.
+</p>
+<p>
+<b>Кэширование.</b> <code>cache</code> читает файл один раз и дальше отвечает из
+памяти; изменившийся на диске файл читается заново, а файл больше мегабайта
+отдаётся так же, как это сделал бы <code>file</code>. При заданном MAX-AGE файл
+не проверяется в течение этого времени - правило уже сообщило клиентам, что
+столько файл можно считать неизменным, - и запрос стоит только копирования
+наружу. Без MAX-AGE каждый запрос делает stat, и изменение подхватывается сразу.
+</p>
+<p>
+<b>Страница блокировки.</b> Сервис, отклоняющий запрос редиректом, может
+отправить клиента на httpsrv, работающий рядом:
+</p><pre>
+auth iponly
+deny * * "pcre:^(ads|track)[0-9]*\.example\.(com|net)$"
+allow *
+proxy -p3128 -i192.168.1.1
+
+flush
+auth iponly
+allow *
+http file * /**  /usr/local/web/blocked.html text/html * * 403
+httpsrv -p8080 -i127.0.0.1
+</pre>
+<p>
+<b>И сайт, и прокси.</b> Запрос может прийти так, как приходит на сайт - путь,
+имя в заголовке Host, - или так, как приходит на прокси: с полным URL, либо, для
+туннеля, с одним именем хоста в CONNECT. Читается и то, и другое. Запрос в форме
+для прокси аутентифицируется через Proxy-Authorization и отклоняется кодом 407,
+как это делает прокси; запрос в форме для сайта - через Authorization и 401. Чем
+он будет обработан, в обоих случаях решают правила.
+</p>
+<p>
+<code>proxypass</code> - правило, которое отвечает, забирая ресурс, поэтому один
+сервис может отдавать своё и проксировать остальное:
+</p><pre>
+auth iponly
+allow *
+http file * /local/** "/usr/local/web/$1"
+http proxypass * /**
+httpsrv -p8080
+</pre>
+<p>
+Правило доступа с перенаправлением на локальный прокси делает то же самое без
+отдельного правила. Цепочка без адреса и означает "локальный прокси", а второй
+<code>allow</code> - то, с чем совпадает сам прокси на своём проходе, так как
+правило с цепочкой второй раз не берётся:
+</p><pre>
+auth iponly
+allow *
+parent 1000 http 0.0.0.0 0
+allow *
+http file * /local/** "/usr/local/web/$1"
+httpsrv -p8080
+</pre>
+<p>
+Аутентификация по той же причине происходит дважды - для сервиса и для прокси, -
+поэтому конфигурация, требующая учётных данных, запрашивает их так, как это
+делает прокси.
+</p>
+<p>
+Правила доступа просматриваются с начала на обоих проходах, и назначение запроса
+известно на втором. На первом сервис отвечает сам за себя, поэтому адрес или порт
+в правиле сопоставляется с адресом, на который подключился клиент; имя из запроса
+сопоставляется на обоих проходах. На втором назначение - то, которое названо в
+запросе, поэтому правила с адресом, портом или именем определяют, что прокси
+разрешено забрать, и определяют это до установления соединения:
+</p><pre>
+auth iponly
+allow *
+parent 1000 http 0.0.0.0 0
+allow * * * 80,443
+deny *
+httpsrv -p8080
+</pre>
+<p>
+До правил доходит всё, забираются только порты 80 и 443, а <code>deny</code>,
+записанный до правила с цепочкой, действует на обоих проходах точно так же. Соединение с сервером сохраняется для следующего запроса и
+закрывается, если следующий запрос идёт в другое место или если сервер за это
+время его закрыл.
+</p>
+<p>
+<b>Соединения.</b> Клиент, обратившийся по HTTP/1.1, получает ответ 1.1, и
+соединение сохраняется для следующего запроса, если он не прислал
+<code>Connection: close</code>; клиенту 1.0 нужно запросить keep-alive явно.
+Соединение сохраняется только тогда, когда длина ответа известна точно - это
+верно для всех операций, кроме страниц администрирования, поэтому они всегда
+последнее, что отдаётся в соединении. Тело запроса, которое сервер не может
+дочитать до конца - присланное chunked или размером больше мегабайта, - тоже
+завершает соединение.
+</p>
+<p>
+<b>Администрирование.</b> Сервис <code>admin</code> - это httpsrv с уже
+объявленными страницами интерфейса администрирования, см.
+<a href="#ADMIN">Администрирование и анализ информации</a>. Правила можно
+добавлять перед ним так же, и они проверяются первыми.
+</p>
+ </p>
  <li><a name="SSLPLUGIN"><i>Как настроить TLS/SSL (https прокси, mTLS)</i></a>
 <p>
 Начиная с версии 0.9.7 поддержка TLS/SSL встроена в 3proxy при компиляции с OpenSSL
@@ -1221,6 +1433,34 @@ pcre_extend deny * 192.168.0.1/16
 <p>
 <b>Примечание:</b> Регулярные выражения не требуют авторизации и не могут заменить
 авторизацию и/или ACL allow/deny.
+</p>
+<p>
+<b>Регулярные выражения в именах хостов:</b> имя хоста в списке назначения
+правила доступа может быть записано регулярным выражением вместо маски, для
+этого используется префикс <code>pcre:</code> (<code>regex:</code> означает то
+же самое). Требуется сборка с поддержкой PCRE, как и для команд
+<code>pcre</code> выше.
+</p><pre>
+&#35; Маска: имя сопоставляется только с начала и с конца
+deny * * *ads.example.com
+
+&#35; Регулярное выражение: всё, что выразимо средствами PCRE
+deny * * "pcre:^(ads|track)[0-9]*\.example\.(com|net)$"
+allow * * "pcre:^(www|api)\.example\.com$"
+</pre>
+<p>
+Перед сопоставлением имя приводится к нижнему регистру, завершающие точки
+удаляются, поэтому шаблоны пишутся в нижнем регистре. Шаблон, оканчивающийся на
+<code>$</code>, нужно взять в кавычки или записать как <code>$$</code>: вне
+кавычек одиночный доллар начинает имя включаемого файла. Имена допустимы только
+в списке назначения (список источника - адреса), и имя проверяется лишь тогда,
+когда оно присутствует в запросе. Маска обходится дешевле и достаточна для
+большинства правил, регулярное выражение сопоставляется на каждый запрос.
+</p>
+<p>
+Тот же префикс и те же шаблоны использует команда <code>http</code> встроенного
+HTTP-сервера - для хоста, на который отвечает правило, и для URL, который оно
+сопоставляет.
 </p>
 
  <li><a name="AUTH"><i>Как ограничить доступ к службе</i></a>

doc/html/man5/3proxy.cfg.5.html

@@ -18,6 +18,14 @@
 <a href="#PCRE FILTERING">PCRE FILTERING</a><br>
 <a href="#PCRE Commands">PCRE Commands</a><br>
 <a href="#PCRE Parameters">PCRE Parameters</a><br>
+<a href="#BUILT IN HTTP SERVER">BUILT IN HTTP SERVER</a><br>
+<a href="#Operations">Operations</a><br>
+<a href="#What a rule adds to the answer">What a rule adds to the answer</a><br>
+<a href="#Patterns">Patterns</a><br>
+<a href="#Both a site and a proxy">Both a site and a proxy</a><br>
+<a href="#Connections">Connections</a><br>
+<a href="#Paths a rule builds">Paths a rule builds</a><br>
+<a href="#Examples">Examples</a><br>
 <a href="#BUGS">BUGS</a><br>
 <a href="#SEE ALSO">SEE ALSO</a><br>
 <a href="#AUTHORS">AUTHORS</a><br>
@@ -77,8 +85,10 @@ characters) is treated as space character (arguments
 delimiter instead of end of command delimiter). Thus,
 include files are only useful to store long single-line
 commands (like userlist, network lists, etc). To use dollar
-sign somewhere in argument it must be quoted. Recursion is
+sign somewhere in argument it must be quoted or doubled:
-not allowed.</p>
+inside quotes a dollar is ordinary text, and <b>$$</b>
+stands for a single dollar and is not read as an include.
+Recursion is not allowed.</p>
 
 <p style="margin-left:9%; margin-top: 1em">Next commands
 start gateway services:</p>
@@ -706,9 +716,24 @@ A.B.C.D - W.X.Y.Z (since 0.8) or CIDRs (W.X.Y.Z/L). Since
 addresses. It&acute;s possible to use a wildmask in the
 beginning and at the end of the hostname, e.g. *badsite.com
 or *badcontent*. The hostname is only checked if a hostname
-is present in the request. Targetportlist may contain ports
+is present in the request. A name written with a
-(X) or port ranges lists (X-Y). For any field * sign means
+<b>pcre:</b> prefix (<b>regex:</b> is the same thing) is a
-ANY. If access list is empty it&acute;s assumed to be <br>
+regular expression instead of a wildmask, in a build with
+PCRE support: <br>
+deny * *
+&quot;pcre:&#710;(ads|track)[0-9]*\.example\.(com|net)$&quot;
+<br>
+The name is lowercased and any trailing dots are removed
+before it is matched, so patterns are written in lower case.
+A pattern ending in <b>$</b> has to be quoted or written
+<b>$$</b>, since a lone dollar outside quotes begins the
+name of a file to include. The same patterns, and the same
+prefix, are used by the <b>http</b> command, see BUILT IN
+HTTP SERVER. Regular expressions are matched per request and
+cost more than a wildmask, which is enough for most rules.
+Targetportlist may contain ports (X) or port ranges lists
+(X-Y). For any field * sign means ANY. If access list is
+empty it&acute;s assumed to be <br>
 allow * <br>
 If access list is not empty last item in access list is
 assumed to be <br>
@@ -1428,8 +1453,17 @@ PCRE_NO_AUTO_CAPTURE, PCRE_NO_UTF8_CHECK, PCRE_AUTO_CALLOUT,
 PCRE_PARTIAL, PCRE_DFA_SHORTEST, PCRE_DFA_RESTART,
 PCRE_FIRSTLINE, PCRE_DUPNAMES, PCRE_NEWLINE_CR,
 PCRE_NEWLINE_LF, PCRE_NEWLINE_CRLF, PCRE_NEWLINE_ANY,
-PCRE_NEWLINE_ANYCRLF, PCRE_BSR_ANYCRLF,
+PCRE_NEWLINE_ANYCRLF, PCRE_BSR_ANYCRLF, PCRE_BSR_UNICODE.
-PCRE_BSR_UNICODE.</p>
+<br>
+These options apply to every pattern the configuration
+compiles, the host patterns of access rules and <b>http</b>
+rules included, so set them before the rules which are to
+use them. <br>
+Regular expressions are not only for these commands: a host
+name in the target list of an access rule, and the host and
+URL of an <b>http</b> rule, take one when it is written with
+a <b>pcre:</b> prefix. See <b>allow</b> and BUILT IN HTTP
+SERVER.</p>
 
 <h3>PCRE Parameters
 <a name="PCRE Parameters"></a>
@@ -1482,6 +1516,264 @@ the connection data. Warning: Regular expressions
 don&rsquo;t require authentication and cannot replace
 authentication and/or allow/deny ACLs.</p>
 
+<h2>BUILT IN HTTP SERVER
+<a name="BUILT IN HTTP SERVER"></a>
+</h2>
+
+
+<p style="margin-left:9%; margin-top: 1em">The
+<b>httpsrv</b> service answers requests itself instead of
+forwarding them. What it does with a request is decided by
+<b>http</b> rules, which are taken in the order they are
+written: the first whose host and URL both match handles the
+request. Rules belong to the service that follows them, the
+way access rules do, and <b>admin</b> is <b>httpsrv</b> with
+a set of rules already in place.</p>
+
+<p style="margin-left:9%; margin-top: 1em"><b>http</b>
+<i>OPERATION HOST URL [PARAMETERS]</i> <br>
+Handle a request for <i>URL</i> on <i>HOST</i> with
+<i>OPERATION</i>. HOST is matched against the Host header,
+URL against the path, with the query string removed.</p>
+
+<h3>Operations
+<a name="Operations"></a>
+</h3>
+
+
+<p style="margin-left:9%; margin-top: 1em"><b>file</b>
+<i>PATH [TYPE [MAX-AGE [HEADERS [CODE]]]]</i> - send the
+file at PATH. The file is handed to the socket by the system
+where it can do that (sendfile, TransmitFile) and read here
+where it cannot, as when the connection carries TLS. The
+arguments after PATH are described below, and each of them
+may be written as <b>*</b> to leave it out. <b><br>
+cache</b> <i>PATH [TYPE [MAX-AGE [HEADERS [CODE]]]]</i> -
+the same, but the file is read into memory on the first
+request and answered from there afterwards. A file that has
+changed on disk is read again, and one larger than a
+megabyte is sent as <b>file</b> would. With a MAX-AGE the
+file is not looked at again for that long: the rule has
+already told clients the file may be treated as unchanged
+for that time, so the server treats its own copy the same
+way and a request costs nothing but the copy out. Without
+one every request stats the file, so a change is picked up
+at once. <b><br>
+reply</b> <i>[CODE [HEADERS]]</i> - answer with a status and
+nothing else. CODE is the status to send, 200 without one. A
+status which carries no body of its own (1xx, 204, 304) is
+sent without a length; anything else is sent with a length
+of zero. <b><br>
+redir</b> <i>[CODE] LOCATION</i> - answer with a redirect.
+CODE is 301 or 302, or any status from 300 to 399; without
+one, 302 is used. <b><br>
+rewrite</b> <i>PATH</i> - change the path of the request and
+hand it to the rules that follow this one. <b><br>
+rewrite_host</b> <i>HOST</i> - the same for the host, which
+decides which of the rules after it match. <b>$1</b> upwards
+stand for what the stars, or the groups, of this
+rule&acute;s host pattern matched, the way they stand for
+those of the URL in a <b>rewrite</b>. What is built has to
+be a host name; the name the client sent is what access
+rules matched and what the log records. <b><br>
+echo</b> - answer with a description of the request: the
+method, path, query, host, and the address and port it came
+from. For testing. <b><br>
+data</b> <i>[size=N] [block=N] [status=N] [chunked=1]
+[delay=N]</i> - answer with generated content of the size
+asked for. For testing. <b><br>
+proxypass</b> - hand the request to the proxy code, which
+fetches it the way <b>proxy</b> would, see BOTH A SITE AND A
+PROXY. <b><br>
+admin</b>, <b>admin_counters</b>, <b>admin_reload</b>,
+<b>admin_services</b> - the pages of the administration
+interface.</p>
+
+<h3>What a rule adds to the answer
+<a name="What a rule adds to the answer"></a>
+</h3>
+
+
+<p style="margin-left:9%; margin-top: 1em"><b>TYPE</b> is
+the content type to answer with. Without it, or with
+<b>*</b>, the type is worked out from the name of the file,
+see <b>http_content_type</b>. <b><br>
+MAX-AGE</b> is a number of seconds, and is sent as
+Cache-Control: max-age. Without it, or with <b>*</b>,
+nothing is said about caching. <b><br>
+HEADERS</b> is one argument holding whole header lines,
+separated by a backslash and an n - the two characters,
+since a configuration line cannot carry a line ending. Each
+becomes a real line ending in the answer. Quote the argument
+if any header holds a space, which they usually do. <b><br>
+CODE</b> is the status to answer with instead of 200, which
+is how a file serves as the body of an error page. <br>
+A rule&rsquo;s headers and MAX-AGE go with whatever status
+that rule asked for. They are not sent with a refusal the
+server itself decided on: a request for a file which is not
+there is answered 404 by the server, not by the rule.
+<b><br>
+file</b> and <b>cache</b> send Last-Modified, and answer a
+request carrying If-Modified-Since with 304 and no body when
+the file has not changed since the time it names. All three
+date formats HTTP allows are read; one which cannot be read
+is treated as no date at all. A rule answering with a CODE
+of its own is answering something other than the file, so it
+is never turned into a 304. <br>
+http file * /err/** &quot;/usr/local/web/404.html&quot;
+text/html * &quot;X-Served: static&quot; 404 <br>
+http reply * /health** 200 &quot;X-Health: ok&quot; <br>
+http reply * /down** 503 &quot;Retry-After: 30&quot;</p>
+
+
+<p style="margin-left:9%; margin-top: 1em"><b>http_content_type</b>
+<i>EXTENSION TYPE</i> <br>
+Answer for a file with that extension with that content
+type, in addition to the types already known. The extension
+may be written with or without its dot. A type named by a
+rule is used whatever this says, and a name the server knows
+nothing about is answered as application/octet-stream. <br>
+http_content_type .webp image/webp</p>
+
+<h3>Patterns
+<a name="Patterns"></a>
+</h3>
+
+
+<p style="margin-left:9%; margin-top: 1em">A URL is matched
+with stars, or with a regular expression when it carries a
+<b>pcre:</b> prefix (<b>regex:</b> is taken as well). A host
+is matched the way an access list matches one, and takes the
+same prefix. <b><br>
+*</b> stands for any run of characters within one element of
+the path: it does not cross a <b>/</b>, so a rule cannot
+reach into a directory it did not name. <b><br>
+**</b> crosses them, and is what a rule which should match
+everything below a point, or everything at all, is written
+with. <br>
+Each star, and each group of a regular expression, is
+remembered in the order it appears. <b>$1</b> upwards stand
+for them in the path or location a rule builds, and
+<b>$0</b> for the whole request path. <br>
+Outside quotes a dollar begins the name of a file to
+include, so an argument holding one - a path or location
+built with <b>$1</b>, a regular expression anchored with
+<b>$</b> - is written in quotes. <b>$$</b> stands for a
+single dollar and is not read as an include either, which is
+how a dollar reaches a rule as text.</p>
+
+<h3>Both a site and a proxy
+<a name="Both a site and a proxy"></a>
+</h3>
+
+
+<p style="margin-left:9%; margin-top: 1em">A request may
+arrive the way it arrives at a site, naming a path and a
+host in the Host header, or the way it arrives at a proxy,
+naming the whole URL, or, for a tunnel, the host alone with
+CONNECT. Both are read. A request in the proxy form
+authenticates with Proxy-Authorization and is refused with
+407, as a proxy refuses one; a request in the site form uses
+Authorization and 401. <br>
+What answers a request is still decided by the rules.
+<b>proxypass</b> is the rule which answers by fetching, so a
+service can serve what it has and proxy the rest: <br>
+http file * /local/** &quot;/usr/local/web/$1&quot; <br>
+http proxypass * /** <br>
+httpsrv -p8080 <br>
+The same happens without a rule for it where an access rule
+redirects to the local proxy, which is written as a chain of
+no address: the rules are asked first, and a request none of
+them answers is fetched. <br>
+allow * <br>
+parent 1000 http 0.0.0.0 0 <br>
+allow * <br>
+The second <b>allow</b> is what the proxy matches on the
+pass it makes itself: a rule carrying the chain is not taken
+twice. Authentication happens twice for the same reason,
+once for the service and once for the proxy, so a
+configuration asking for credentials asks for them as a
+proxy does. <br>
+The access rules are read from the top on both passes, and
+it is the second pass which describes where the request is
+going. On the first one the service is answering for itself,
+so the destination an address or a port is matched against
+is the address the client connected to; the name from the
+request is matched on both. On the second the destination is
+the one the request names, so rules written with an address,
+a port or a name decide what the proxy is allowed to fetch,
+and they decide it before the connection is made: <br>
+allow * <br>
+parent 1000 http 0.0.0.0 0 <br>
+allow * * * 80,443 <br>
+deny * <br>
+Everything reaches the rules, and only ports 80 and 443 are
+fetched. A rule before the one carrying the chain applies on
+both passes just the same, so a <b>deny</b> written there
+stops the request as well. <br>
+The connection to the server is kept for the request after
+it, and closed when the request after it goes somewhere
+else, or when the server has closed it in the meantime. A
+tunnel is fetched by the proxy code as well, which means the
+connection carrying it belongs to that request alone.</p>
+
+<h3>Connections
+<a name="Connections"></a>
+</h3>
+
+
+<p style="margin-left:9%; margin-top: 1em">An answer is
+sent as HTTP/1.1 to a client which asked in HTTP/1.1, and
+the connection is kept for the next request unless the
+client sent <b>Connection: close</b>. A 1.0 client gets a
+1.0 answer, and the connection is kept only when it asked
+with <b>Connection: keep-alive</b>. <br>
+The connection is kept only when what was sent is framed
+exactly: every operation but the administration pages states
+a length, or sends a chunked body a 1.1 client can read, so
+the pages of <b>admin</b> are always the last thing on a
+connection. A request body which cannot be read to its end
+ends the connection as well: one sent with
+<b>Transfer-Encoding</b>, which this server does not read,
+and one longer than a megabyte, which it will not.</p>
+
+<h3>Paths a rule builds
+<a name="Paths a rule builds"></a>
+</h3>
+
+
+<p style="margin-left:9%; margin-top: 1em">The path a rule
+builds is used as it is, so it is refused rather than
+corrected when it is not a plain full path. A relative path
+is refused: it would be read against whatever directory the
+service happens to be in. So is one holding <b>.</b> or
+<b>..</b> as an element, a carriage return, a newline or a
+star. On Windows a path must name a drive or a share, and is
+converted to the extended \\?\ form and opened through the
+wide interface, so a long path works. <br>
+A request is checked before any of this: a path which
+decodes to one leaving the tree is refused outright.</p>
+
+<h3>Examples
+<a name="Examples"></a>
+</h3>
+
+
+<p style="margin-left:9%; margin-top: 1em">http file
+example.com /my/webpath/*.html
+&quot;/usr/local/web/$1.html&quot; <br>
+http cache example.com
+&quot;pcre:&#710;/(.*)/pic/(.*).(gif|jpeg)$&quot;
+&quot;/usr/local/web/picts/$1/$2.$3&quot; <br>
+http redir * /old/** 301 &quot;https://example.org/$1&quot;
+<br>
+http rewrite * /alias/** &quot;/w/$1&quot; <br>
+http rewrite_host *.old.example **
+&quot;$1.new.example&quot; <br>
+http file * /static/** &quot;/usr/local/web/static/$1&quot;
+<br>
+httpsrv -p8080</p>
+
 <h2>BUGS
 <a name="BUGS"></a>
 </h2>

man/3proxy.cfg.5

@@ -39,7 +39,9 @@ For included file <CR> (end of line characters) is treated as space character
 (arguments delimiter instead of end of command delimiter). 
 Thus, include files are only useful to store long single-line commands
 (like userlist, network lists, etc).
-To use dollar sign somewhere in argument it must be quoted. 
+To use dollar sign somewhere in argument it must be quoted or doubled: inside
+quotes a dollar is ordinary text, and \fB$$\fR stands for a single dollar and is
+not read as an include.
 Recursion is not allowed.
 
 .br
@@ -743,6 +745,17 @@ Since 0.6, the targetlist may also contain host names,
 instead of addresses. It\'s possible to use a wildmask in
 the beginning and at the end of the hostname, e.g. *badsite.com or *badcontent*.
 The hostname is only checked if a hostname is present in the request.
+A name written with a \fBpcre:\fR prefix (\fBregex:\fR is the same thing) is a
+regular expression instead of a wildmask, in a build with PCRE support:
+.br
+ deny * * "pcre:^(ads|track)[0-9]*\\.example\\.(com|net)$"
+.br
+ The name is lowercased and any trailing dots are removed before it is matched,
+so patterns are written in lower case. A pattern ending in \fB$\fR has to be
+quoted or written \fB$$\fR, since a lone dollar outside quotes begins the name
+of a file to include. The same patterns, and the same prefix, are used by the
+\fBhttp\fR command, see BUILT IN HTTP SERVER. Regular expressions are matched
+per request and cost more than a wildmask, which is enough for most rules.
 Targetportlist may contain ports (X) or port ranges lists (X-Y). For any field *
 sign means ANY. If access list is empty it\'s assumed to be
 .br
@@ -1511,6 +1524,15 @@ PCRE_NOTEMPTY, PCRE_UTF8, PCRE_NO_AUTO_CAPTURE, PCRE_NO_UTF8_CHECK, PCRE_AUTO_CA
 PCRE_PARTIAL, PCRE_DFA_SHORTEST, PCRE_DFA_RESTART, PCRE_FIRSTLINE, PCRE_DUPNAMES,
 PCRE_NEWLINE_CR, PCRE_NEWLINE_LF, PCRE_NEWLINE_CRLF, PCRE_NEWLINE_ANY, PCRE_NEWLINE_ANYCRLF,
 PCRE_BSR_ANYCRLF, PCRE_BSR_UNICODE.
+.br
+ These options apply to every pattern the configuration compiles, the host
+patterns of access rules and \fBhttp\fR rules included, so set them before the
+rules which are to use them.
+.br
+ Regular expressions are not only for these commands: a host name in the target
+list of an access rule, and the host and URL of an \fBhttp\fR rule, take one
+when it is written with a \fBpcre:\fR prefix. See \fBallow\fR and BUILT IN
+HTTP SERVER.
 
 .SS PCRE Parameters
 TYPE - type of filtered data (comma-delimited list):
@@ -1551,6 +1573,226 @@ matched if the ACL matches the connection data.
 Warning: Regular expressions don't require authentication and cannot replace
 authentication and/or allow/deny ACLs.
 
+.SH BUILT IN HTTP SERVER
+The \fBhttpsrv\fR service answers requests itself instead of forwarding them.
+What it does with a request is decided by \fBhttp\fR rules, which are taken in
+the order they are written: the first whose host and URL both match handles the
+request. Rules belong to the service that follows them, the way access rules do,
+and \fBadmin\fR is \fBhttpsrv\fR with a set of rules already in place.
+
+.BR http
+\fIOPERATION HOST URL [PARAMETERS]\fR
+.br
+ Handle a request for \fIURL\fR on \fIHOST\fR with \fIOPERATION\fR. HOST is
+matched against the Host header, URL against the path, with the query string
+removed.
+
+.SS Operations
+.br
+ \fBfile\fR \fIPATH [TYPE [MAX-AGE [HEADERS [CODE]]]]\fR - send the file at PATH.
+The file is handed to the socket by the system where it can do that (sendfile,
+TransmitFile) and read here where it cannot, as when the connection carries TLS.
+The arguments after PATH are described below, and each of them may be written as
+\fB*\fR to leave it out.
+.br
+ \fBcache\fR \fIPATH [TYPE [MAX-AGE [HEADERS [CODE]]]]\fR - the same, but the
+file is read into memory on the first request and answered from there afterwards.
+A file that has changed on disk is read again, and one larger than a megabyte is
+sent as \fBfile\fR would. With a MAX-AGE the file is not looked at again for
+that long: the rule has already told clients the file may be treated as
+unchanged for that time, so the server treats its own copy the same way and a
+request costs nothing but the copy out. Without one every request stats the
+file, so a change is picked up at once.
+.br
+ \fBreply\fR \fI[CODE [HEADERS]]\fR - answer with a status and nothing else.
+CODE is the status to send, 200 without one. A status which carries no body of
+its own (1xx, 204, 304) is sent without a length; anything else is sent with a
+length of zero.
+.br
+ \fBredir\fR \fI[CODE] LOCATION\fR - answer with a redirect. CODE is 301 or 302,
+or any status from 300 to 399; without one, 302 is used.
+.br
+ \fBrewrite\fR \fIPATH\fR - change the path of the request and hand it to the
+rules that follow this one.
+.br
+ \fBrewrite_host\fR \fIHOST\fR - the same for the host, which decides which of
+the rules after it match. \fB$1\fR upwards stand for what the stars, or the
+groups, of this rule\'s host pattern matched, the way they stand for those of
+the URL in a \fBrewrite\fR. What is built has to be a host name; the name the
+client sent is what access rules matched and what the log records.
+.br
+ \fBecho\fR - answer with a description of the request: the method, path, query,
+host, and the address and port it came from. For testing.
+.br
+ \fBdata\fR \fI[size=N] [block=N] [status=N] [chunked=1] [delay=N]\fR - answer
+with generated content of the size asked for. For testing.
+.br
+ \fBproxypass\fR - hand the request to the proxy code, which fetches it the
+way \fBproxy\fR would, see BOTH A SITE AND A PROXY.
+.br
+ \fBadmin\fR, \fBadmin_counters\fR, \fBadmin_reload\fR, \fBadmin_services\fR -
+the pages of the administration interface.
+
+.SS What a rule adds to the answer
+\fBTYPE\fR is the content type to answer with. Without it, or with \fB*\fR, the
+type is worked out from the name of the file, see \fBhttp_content_type\fR.
+.br
+ \fBMAX-AGE\fR is a number of seconds, and is sent as Cache-Control: max-age.
+Without it, or with \fB*\fR, nothing is said about caching.
+.br
+ \fBHEADERS\fR is one argument holding whole header lines, separated by a
+backslash and an n \- the two characters, since a configuration line cannot
+carry a line ending. Each becomes a real line ending in the answer. Quote the
+argument if any header holds a space, which they usually do.
+.br
+ \fBCODE\fR is the status to answer with instead of 200, which is how a file
+serves as the body of an error page.
+.br
+ A rule's headers and MAX-AGE go with whatever status that rule asked for. They
+are not sent with a refusal the server itself decided on: a request for a file
+which is not there is answered 404 by the server, not by the rule.
+.br
+ \fBfile\fR and \fBcache\fR send Last-Modified, and answer a request carrying
+If-Modified-Since with 304 and no body when the file has not changed since the
+time it names. All three date formats HTTP allows are read; one which cannot be
+read is treated as no date at all. A rule answering with a CODE of its own is
+answering something other than the file, so it is never turned into a 304.
+.br
+ http file * /err/** "/usr/local/web/404.html" text/html * "X-Served: static" 404
+.br
+ http reply * /health** 200 "X-Health: ok"
+.br
+ http reply * /down** 503 "Retry-After: 30"
+
+.BR http_content_type
+\fIEXTENSION TYPE\fR
+.br
+ Answer for a file with that extension with that content type, in addition to
+the types already known. The extension may be written with or without its dot.
+A type named by a rule is used whatever this says, and a name the server knows
+nothing about is answered as application/octet-stream.
+.br
+ http_content_type .webp image/webp
+
+.SS Patterns
+A URL is matched with stars, or with a regular expression when it carries a
+\fBpcre:\fR prefix (\fBregex:\fR is taken as well). A host is matched the way an
+access list matches one, and takes the same prefix.
+.br
+ \fB*\fR stands for any run of characters within one element of the path: it
+does not cross a \fB/\fR, so a rule cannot reach into a directory it did not
+name.
+.br
+ \fB**\fR crosses them, and is what a rule which should match everything below a
+point, or everything at all, is written with.
+.br
+ Each star, and each group of a regular expression, is remembered in the order
+it appears. \fB$1\fR upwards stand for them in the path or location a rule
+builds, and \fB$0\fR for the whole request path.
+.br
+ Outside quotes a dollar begins the name of a file to include, so an argument
+holding one \- a path or location built with \fB$1\fR, a regular expression
+anchored with \fB$\fR \- is written in quotes. \fB$$\fR stands for a single
+dollar and is not read as an include either, which is how a dollar reaches a
+rule as text.
+
+.SS Both a site and a proxy
+A request may arrive the way it arrives at a site, naming a path and a host in
+the Host header, or the way it arrives at a proxy, naming the whole URL, or, for
+a tunnel, the host alone with CONNECT. Both are read. A request in the proxy
+form authenticates with Proxy-Authorization and is refused with 407, as a proxy
+refuses one; a request in the site form uses Authorization and 401.
+.br
+ What answers a request is still decided by the rules. \fBproxypass\fR is the
+rule which answers by fetching, so a service can serve what it has and proxy the
+rest:
+.br
+ http file * /local/** "/usr/local/web/$1"
+.br
+ http proxypass * /**
+.br
+ httpsrv -p8080
+.br
+ The same happens without a rule for it where an access rule redirects to the
+local proxy, which is written as a chain of no address: the rules are asked
+first, and a request none of them answers is fetched.
+.br
+ allow *
+.br
+ parent 1000 http 0.0.0.0 0
+.br
+ allow *
+.br
+ The second \fBallow\fR is what the proxy matches on the pass it makes itself:
+a rule carrying the chain is not taken twice. Authentication happens twice for
+the same reason, once for the service and once for the proxy, so a configuration
+asking for credentials asks for them as a proxy does.
+.br
+ The access rules are read from the top on both passes, and it is the second
+pass which describes where the request is going. On the first one the service is
+answering for itself, so the destination an address or a port is matched against
+is the address the client connected to; the name from the request is matched on
+both. On the second the destination is the one the request names, so rules
+written with an address, a port or a name decide what the proxy is allowed to
+fetch, and they decide it before the connection is made:
+.br
+ allow *
+.br
+ parent 1000 http 0.0.0.0 0
+.br
+ allow * * * 80,443
+.br
+ deny *
+.br
+ Everything reaches the rules, and only ports 80 and 443 are fetched. A rule
+before the one carrying the chain applies on both passes just the same, so a
+\fBdeny\fR written there stops the request as well.
+.br
+ The connection to the server is kept for the request after it, and closed when
+the request after it goes somewhere else, or when the server has closed it in
+the meantime. A tunnel is fetched by the proxy code as well, which means the
+connection carrying it belongs to that request alone.
+
+.SS Connections
+An answer is sent as HTTP/1.1 to a client which asked in HTTP/1.1, and the
+connection is kept for the next request unless the client sent
+\fBConnection: close\fR. A 1.0 client gets a 1.0 answer, and the connection is
+kept only when it asked with \fBConnection: keep-alive\fR.
+.br
+ The connection is kept only when what was sent is framed exactly: every
+operation but the administration pages states a length, or sends a chunked body
+a 1.1 client can read, so the pages of \fBadmin\fR are always the last thing on
+a connection. A request body which cannot be read to its end ends the connection
+as well: one sent with \fBTransfer-Encoding\fR, which this server does not read,
+and one longer than a megabyte, which it will not.
+
+.SS Paths a rule builds
+The path a rule builds is used as it is, so it is refused rather than corrected
+when it is not a plain full path. A relative path is refused: it would be read
+against whatever directory the service happens to be in. So is one holding
+\fB.\fR or \fB..\fR as an element, a carriage return, a newline or a star. On
+Windows a path must name a drive or a share, and is converted to the extended
+\\\\?\\ form and opened through the wide interface, so a long path works.
+.br
+ A request is checked before any of this: a path which decodes to one leaving
+the tree is refused outright.
+
+.SS Examples
+.br
+ http file example.com /my/webpath/*.html "/usr/local/web/$1.html"
+.br
+ http cache example.com "pcre:^/(.*)/pic/(.*)\.(gif|jpeg)$" "/usr/local/web/picts/$1/$2.$3"
+.br
+ http redir * /old/** 301 "https://example.org/$1"
+.br
+ http rewrite * /alias/** "/w/$1"
+.br
+ http rewrite_host *.old.example ** "$1.new.example"
+.br
+ http file * /static/** "/usr/local/web/static/$1"
+.br
+ httpsrv -p8080
+
 .SH BUGS
 Report all bugs to
 .BR 3proxy@3proxy.org

src/3proxy.c

@@ -16,6 +16,9 @@ void pcre_install(void);
 #ifdef WITH_TRANSPARENT
 void transparent_install(void);
 #endif
+#ifdef WITH_HTTPSRV
+void httpsrv_init(void);
+#endif
 #ifndef _WIN32
 #include <sys/resource.h>
 #ifndef NOPLUGINS
@@ -535,6 +538,9 @@ int WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPWSTR lpCmdLine, int
 #ifdef WITH_TRANSPARENT
   transparent_install();
 #endif
+#ifdef WITH_HTTPSRV
+  httpsrv_init();
+#endif
 
   freeconf(&conf);
   initcommands();

src/acl.c

@@ -8,6 +8,247 @@
 
 #include "proxy.h"
 
+/* The pattern engine lives here rather than in common.c: common.c is linked
+   into the standalone binaries as well, and those carry neither the regular
+   expression code this calls nor a use for a host pattern. */
+/* Host lists in access rules have always accepted name, name*, *name and
+   *name*, with the leading and trailing star recorded as a match type rather
+   than kept in the string. The parser and the comparison are here so that
+   anything else matching a name against a pattern - the http command, and
+   whatever replaces the star with a regular expression later - behaves the same
+   way and gains the same syntax at the same time.
+ */
+/* A pattern written as a regular expression carries a prefix. Both spellings
+   are taken so a configuration reads the way its author thinks of it. */
+static unsigned char * regexprefix(unsigned char *arg)
+{
+	if(!strncmp((char *)arg, "pcre:", 5)) return arg + 5;
+	if(!strncmp((char *)arg, "regex:", 6)) return arg + 6;
+	return NULL;
+}
+
+/* Shared by every pattern the configuration can carry. Returns 0 on success. */
+static int compileregex(struct hostname *h, unsigned char *pattern)
+{
+#ifdef WITH_PCRE
+	char err[256];
+
+	h->re = pcre_pattern_compile(pattern, err, sizeof(err));
+	if(!h->re){
+		fprintf(stderr, "Bad regular expression '%s': %s\n", pattern, err);
+		return 1;
+	}
+	h->matchtype = MATCHREGEX;
+	h->name = (unsigned char *)strdup((char *)pattern);
+	return h->name? 0 : 1;
+#else
+	fprintf(stderr, "Regular expression '%s' needs a build with PCRE\n", pattern);
+	return 1;
+#endif
+}
+
+int parsepattern(struct hostname *h, unsigned char *arg)
+{
+	int arglen;
+	unsigned char *pattern;
+
+	h->re = NULL;
+	if((pattern = regexprefix(arg))) return compileregex(h, pattern);
+
+	arglen = (int)strlen((char *)arg);
+	h->matchtype = 3;
+	pattern = arg;
+
+	if(arglen && pattern[arglen-1] == '*'){
+		arglen--;
+		pattern[arglen] = 0;
+		h->matchtype ^= MATCHEND;
+	}
+	if(arglen && pattern[0] == '*'){
+		pattern++;
+		arglen--;
+		h->matchtype ^= MATCHBEGIN;
+	}
+
+	h->name = (unsigned char *)strdup((char *)pattern);
+	return h->name? 0 : 1;
+}
+
+/* Matches str against a pattern and reports the part a star stood for. Where a
+   pattern has a star at both ends the trailing one is reported, since that is
+   the part following the text that was matched. An exact pattern leaves an
+   empty span. */
+int patternmatchpos(const struct hostname *h, const unsigned char *str, int *start, int *len)
+{
+	int lname, lstr, pos = 0, match = 0;
+	char *found;
+
+	if(!h->name || !str) return 0;
+	if(h->matchtype == MATCHREGEX || h->matchtype == MATCHGLOB){
+		struct capture caps[MAXCAPTURES];
+
+		if(!patternmatchcaps(h, str, caps, NULL)) return 0;
+		if(start) *start = caps[1].start;
+		if(len) *len = caps[1].len;
+		return 1;
+	}
+
+	lname = (int)strlen((char *)h->name);
+	lstr = (int)strlen((char *)str);
+
+	switch(h->matchtype){
+		case 0:
+#ifndef _WIN32
+			found = strcasestr((char *)str, (char *)h->name);
+#else
+			found = strstr((char *)str, (char *)h->name);
+#endif
+			if(found){
+				match = 1;
+				pos = (int)(found - (char *)str) + lname;
+			}
+			break;
+
+		case 1:
+			if(!strncasecmp((char *)str, (char *)h->name, lname)){
+				match = 1;
+				pos = lname;
+			}
+			break;
+
+		case 2:
+			if(lstr >= lname &&
+			   !strncasecmp((char *)str + (lstr - lname), (char *)h->name, lname)){
+				match = 1;
+				pos = 0;
+				if(start) *start = 0;
+				if(len) *len = lstr - lname;
+				return 1;
+			}
+			break;
+
+		default:
+			if(!strcasecmp((char *)str, (char *)h->name)){
+				match = 1;
+				pos = lstr;
+			}
+			break;
+	}
+
+	if(!match) return 0;
+
+	if(start) *start = pos;
+	if(len) *len = lstr - pos;
+	return 1;
+}
+
+int patternmatch(const struct hostname *h, const unsigned char *str)
+{
+	return patternmatchcaps(h, str, NULL, NULL);
+}
+
+/* Match a glob, recording what each star stood for.
+
+   A single star stands for any run of characters within one element of the
+   path, so it stops at a slash; a double star crosses them. Stars are
+   numbered in the order they appear, which is how a template refers to them.
+ */
+static int globmatch(const unsigned char *pat, const unsigned char *str,
+	const unsigned char *subject, struct capture *caps, int maxcaps, int star)
+{
+	while(*pat){
+		if(*pat == '*'){
+			int crosses = (pat[1] == '*');
+			const unsigned char *rest = pat + (crosses? 2 : 1);
+			int len;
+
+			for(len = 0; ; len++){
+				if(star < maxcaps && caps){
+					caps[star].start = (int)(str - subject);
+					caps[star].len = len;
+				}
+				if(globmatch(rest, str + len, subject, caps, maxcaps, star + 1)) return 1;
+				if(!str[len]) return 0;
+				if(!crosses && str[len] == '/') return 0;
+			}
+		}
+		if(*pat != *str) return 0;
+		pat++;
+		str++;
+	}
+	return *str == 0;
+}
+
+/* Match a pattern of any kind and report what its stars or groups stood for.
+   caps may be NULL when only the yes or no answer is wanted. */
+int patternmatchcaps(const struct hostname *h, const unsigned char *str,
+	struct capture *caps, int *ncaps)
+{
+	int n = 0;
+
+	if(ncaps) *ncaps = 0;
+	if(!h || !str) return 0;
+
+	if(h->matchtype == MATCHREGEX){
+#ifdef WITH_PCRE
+		struct capture local[MAXCAPTURES];
+
+		n = pcre_pattern_match(h->re, str, caps? caps : local, MAXCAPTURES);
+		if(ncaps) *ncaps = n;
+		return n > 0;
+#else
+		return 0;
+#endif
+	}
+
+	if(h->matchtype == MATCHGLOB){
+		struct capture local[MAXCAPTURES];
+		struct capture *use = caps? caps : local;
+		int i;
+
+		for(i = 0; i < MAXCAPTURES; i++){
+			use[i].start = 0;
+			use[i].len = 0;
+		}
+		use[0].start = 0;
+		use[0].len = (int)strlen((char *)str);
+		if(!h->name) return 0;
+		if(!globmatch(h->name, str, str, use, MAXCAPTURES, 1)) return 0;
+		if(ncaps){
+			for(n = MAXCAPTURES - 1; n > 0 && !use[n].len && !use[n].start; n--);
+			*ncaps = n + 1;
+		}
+		return 1;
+	}
+
+	/* the star at one end or both, as an access rule has always written it */
+	if(caps){
+		int start = 0, len = 0;
+
+		if(!patternmatchpos(h, str, &start, &len)) return 0;
+		caps[0].start = 0;
+		caps[0].len = (int)strlen((char *)str);
+		caps[1].start = start;
+		caps[1].len = len;
+		if(ncaps) *ncaps = 2;
+		return 1;
+	}
+	return patternmatchpos(h, str, NULL, NULL);
+}
+
+/* A URL in an http rule: stars anywhere, or a regular expression. */
+int parsepathpattern(struct hostname *h, unsigned char *arg)
+{
+	unsigned char *pattern;
+
+	h->re = NULL;
+	if((pattern = regexprefix(arg))) return compileregex(h, pattern);
+
+	h->matchtype = MATCHGLOB;
+	h->name = (unsigned char *)strdup((char *)arg);
+	return h->name? 0 : 1;
+}
+
 int IPInentry(struct sockaddr *sa, struct iplist *ipentry){
 	int addrlen;
 	unsigned char *ip, *ipf, *ipt;

src/auth.c

@@ -20,8 +20,9 @@ int alwaysauth(struct clientparam * param){
 	if(conf.connlimiter && !param->connlim  && startconnlims(param)) return 10;
 #ifdef WITH_HTTPSRV
 	/* The http server answers the request itself, so authorization must not
-	   try to reach a destination that does not exist. */
+	   try to reach a destination that does not exist. A request it has handed
-	res = (param->srv->service == S_HTTPSRV)? 0 : doconnect(param);
+	   to another child does have one, and that child needs it opened. */
+	res = (param->srv->service == S_HTTPSRV && !param->onerequest)? 0 : doconnect(param);
 #else
 	res = doconnect(param);
 #endif

src/common.c

@@ -810,102 +810,6 @@ int bindwithrange(struct clientparam *param, SOCKET sock, PROXYSOCKADDRTYPE *sa,
 	return param->srv->so._bind(param->sostate, sock, (struct sockaddr *)sa, SASIZE(sa));
 }
 
-/* Host lists in access rules have always accepted name, name*, *name and
-   *name*, with the leading and trailing star recorded as a match type rather
-   than kept in the string. The parser and the comparison are here so that
-   anything else matching a name against a pattern - the http command, and
-   whatever replaces the star with a regular expression later - behaves the same
-   way and gains the same syntax at the same time.
- */
-int parsepattern(struct hostname *h, unsigned char *arg)
-{
-	int arglen;
-	unsigned char *pattern;
-
-	arglen = (int)strlen((char *)arg);
-	h->matchtype = 3;
-	pattern = arg;
-
-	if(arglen && pattern[arglen-1] == '*'){
-		arglen--;
-		pattern[arglen] = 0;
-		h->matchtype ^= MATCHEND;
-	}
-	if(arglen && pattern[0] == '*'){
-		pattern++;
-		arglen--;
-		h->matchtype ^= MATCHBEGIN;
-	}
-
-	h->name = (unsigned char *)strdup((char *)pattern);
-	return h->name? 0 : 1;
-}
-
-/* Matches str against a pattern and reports the part a star stood for. Where a
-   pattern has a star at both ends the trailing one is reported, since that is
-   the part following the text that was matched. An exact pattern leaves an
-   empty span. */
-int patternmatchpos(const struct hostname *h, const unsigned char *str, int *start, int *len)
-{
-	int lname, lstr, pos = 0, match = 0;
-	char *found;
-
-	if(!h->name || !str) return 0;
-
-	lname = (int)strlen((char *)h->name);
-	lstr = (int)strlen((char *)str);
-
-	switch(h->matchtype){
-		case 0:
-#ifndef _WIN32
-			found = strcasestr((char *)str, (char *)h->name);
-#else
-			found = strstr((char *)str, (char *)h->name);
-#endif
-			if(found){
-				match = 1;
-				pos = (int)(found - (char *)str) + lname;
-			}
-			break;
-
-		case 1:
-			if(!strncasecmp((char *)str, (char *)h->name, lname)){
-				match = 1;
-				pos = lname;
-			}
-			break;
-
-		case 2:
-			if(lstr >= lname &&
-			   !strncasecmp((char *)str + (lstr - lname), (char *)h->name, lname)){
-				match = 1;
-				pos = 0;
-				if(start) *start = 0;
-				if(len) *len = lstr - lname;
-				return 1;
-			}
-			break;
-
-		default:
-			if(!strcasecmp((char *)str, (char *)h->name)){
-				match = 1;
-				pos = lstr;
-			}
-			break;
-	}
-
-	if(!match) return 0;
-
-	if(start) *start = pos;
-	if(len) *len = lstr - pos;
-	return 1;
-}
-
-int patternmatch(const struct hostname *h, const unsigned char *str)
-{
-	return patternmatchpos(h, str, NULL, NULL);
-}
-
 int scanaddr(const unsigned char *s, uint32_t * ip, uint32_t * mask) {
 	unsigned d1, d2, d3, d4, m;
 	int res;

src/conf.c

@@ -9,7 +9,7 @@
 #include "proxy.h"
 
 #ifdef WITH_HTTPSRV
-static int addhttprule(char *host, char *url, char *op, char *params);
+static int addhttprule(char *op, char *host, char *url, char *params);
 #endif
 #include "mdhash.h"
 #ifdef WITH_SSL
@@ -264,10 +264,10 @@ static int h_proxy(int argc, unsigned char ** argv){
 	else if(!strcmp((char *)argv[0], "admin")) {
 		/* The same service as httpsrv, with the administration pages
 		   declared for it. */
-		if(addhttprule("*", "/C*", "admin_counters", NULL) ||
+		if(addhttprule("admin_counters", "*", "/C*", NULL) ||
-		   addhttprule("*", "/R", "admin_reload", NULL) ||
+		   addhttprule("admin_reload", "*", "/R", NULL) ||
-		   addhttprule("*", "/S*", "admin_services", NULL) ||
+		   addhttprule("admin_services", "*", "/S*", NULL) ||
-		   addhttprule("*", "*", "admin", NULL)){
+		   addhttprule("admin", "*", "**", NULL)){
 			fprintf(stderr, "Failed to declare the admin pages, line %d\n", linenum);
 			return 1;
 		}
@@ -802,8 +802,57 @@ struct redirdesc redirs[] = {
 };
 
 #ifdef WITH_HTTPSRV
+/* Headers a rule adds are written as one argument, the lines separated by a
+   backslash and an n, because a configuration line cannot hold a line ending.
+   Those two characters become a real one here. A line ending which reached the
+   argument as itself is dropped: what goes on the wire is decided here and not
+   by whatever produced the string. */
+static unsigned char * parsehdrs(const unsigned char *arg)
+{
+	unsigned char *out, *o;
+	const unsigned char *p;
+	size_t len = strlen((char *)arg);
+
+	out = malloc(len * 2 + 3);
+	if(!out) return NULL;
+	for(p = arg, o = out; *p; p++){
+		if(*p == '\\' && p[1] == 'n'){
+			*o++ = '\r';
+			*o++ = '\n';
+			p++;
+			continue;
+		}
+		if(*p == '\r' || *p == '\n') continue;
+		*o++ = *p;
+	}
+	if(o == out || o[-1] != '\n'){
+		*o++ = '\r';
+		*o++ = '\n';
+	}
+	*o = 0;
+	return out;
+}
+
+/* An optional argument which a star, or nothing at all, leaves at its
+   default. */
+static int optnum(int argc, unsigned char **argv, int at, int def)
+{
+	if(argc <= at || !strcmp((char *)argv[at], "*")) return def;
+	return atoi((char *)argv[at]);
+}
+
+static void freehttprule(struct httprule *rule)
+{
+	if(rule->host.name) free(rule->host.name);
+	if(rule->url.name) free(rule->url.name);
+	if(rule->params) free(rule->params);
+	if(rule->ctype) free(rule->ctype);
+	if(rule->hdrs) free(rule->hdrs);
+	free(rule);
+}
+
 /* Installs one rule from code, for the pages a service predefines. */
-static int addhttprule(char *host, char *url, char *op, char *params)
+static int addhttprule(char *op, char *host, char *url, char *params)
 {
     struct httprule *rule, *tail;
     unsigned char hostbuf[64], urlbuf[128];
@@ -811,6 +860,7 @@ static int addhttprule(char *host, char *url, char *op, char *params)
 	rule = malloc(sizeof(struct httprule));
 	if(!rule) return 1;
 	memset(rule, 0, sizeof(struct httprule));
+	rule->maxage = -1;
 
 	rule->op = httpopbyname((unsigned char *)op);
 	if(rule->op < 0){
@@ -820,7 +870,7 @@ static int addhttprule(char *host, char *url, char *op, char *params)
 
 	strcpy((char *)hostbuf, host);
 	strcpy((char *)urlbuf, url);
-	if(parsepattern(&rule->host, hostbuf) || parsepattern(&rule->url, urlbuf)){
+	if(parsepattern(&rule->host, hostbuf) || parsepathpattern(&rule->url, urlbuf)){
 		free(rule->host.name);
 		free(rule);
 		return 1;
@@ -973,9 +1023,10 @@ static int h_http(int argc, unsigned char **argv){
     struct httprule *rule, *tail;
     int op;
 
-	op = httpopbyname(argv[3]);
+	/* http OPERATION HOST URL [PARAMETERS] */
+	op = httpopbyname(argv[1]);
 	if(op < 0){
-		fprintf(stderr, "Unknown http operation: %s line %d\n", argv[3], linenum);
+		fprintf(stderr, "Unknown http operation: %s line %d\n", argv[1], linenum);
 		return(1);
 	}
 
@@ -983,23 +1034,75 @@ static int h_http(int argc, unsigned char **argv){
 	if(!rule) return(21);
 	memset(rule, 0, sizeof(struct httprule));
 	rule->op = op;
+	rule->maxage = -1;
 
-	if(parsepattern(&rule->host, argv[1]) || parsepattern(&rule->url, argv[2])){
+	if(parsepattern(&rule->host, argv[2]) || parsepathpattern(&rule->url, argv[3])){
 		fprintf(stderr, "No memory for http rule, line %d\n", linenum);
 		free(rule->host.name);
 		free(rule);
 		return(21);
 	}
 
-	if(argc > 4){
+	if(argc > 4 && (!strcmp((char *)argv[1], "file") || !strcmp((char *)argv[1], "cache"))){
+		/* PATH [TYPE [MAX-AGE [HEADERS [CODE]]]]. A star, or nothing,
+		   leaves each of them out: the type is worked out from the name,
+		   nothing is said about caching, no headers are added and the
+		   answer is the usual 200. */
 		rule->params = (unsigned char *)strdup((char *)argv[4]);
-		if(!rule->params){
+		if(argc > 5 && strcmp((char *)argv[5], "*"))
-			free(rule->host.name);
+			rule->ctype = (unsigned char *)strdup((char *)argv[5]);
-			free(rule->url.name);
+		rule->maxage = optnum(argc, argv, 6, -1);
-			free(rule);
+		if(argc > 7 && strcmp((char *)argv[7], "*"))
+			rule->hdrs = parsehdrs(argv[7]);
+		rule->code = optnum(argc, argv, 8, 0);
+		if(!rule->params
+		   || (argc > 5 && strcmp((char *)argv[5], "*") && !rule->ctype)
+		   || (argc > 7 && strcmp((char *)argv[7], "*") && !rule->hdrs)){
+			freehttprule(rule);
 			return(21);
 		}
 	}
+	else if(!strcmp((char *)argv[1], "reply")){
+		/* CODE [HEADERS], and no body at all. */
+		rule->code = optnum(argc, argv, 4, 200);
+		if(argc > 5 && strcmp((char *)argv[5], "*")){
+			rule->hdrs = parsehdrs(argv[5]);
+			if(!rule->hdrs){
+				freehttprule(rule);
+				return(21);
+			}
+		}
+	}
+	else if(argc > 4){
+		/* What follows the URL belongs to the operation, and an operation
+		   such as redir reads more than one word of it. */
+		int i, len = 0;
+
+		for(i = 4; i < argc; i++) len += (int)strlen((char *)argv[i]) + 1;
+		rule->params = malloc(len);
+		if(rule->params){
+			int at = 0;
+
+			for(i = 4; i < argc; i++)
+				at += sprintf((char *)rule->params + at, "%s%s",
+					i > 4? " " : "", argv[i]);
+		}
+		if(!rule->params){
+			freehttprule(rule);
+			return(21);
+		}
+	}
+
+	if(rule->code && (rule->code < 100 || rule->code > 599)){
+		fprintf(stderr, "Wrong http status: %d line %d\n", rule->code, linenum);
+		freehttprule(rule);
+		return(1);
+	}
+	if(rule->maxage < -1){
+		fprintf(stderr, "Wrong max-age, line %d\n", linenum);
+		freehttprule(rule);
+		return(1);
+	}
 
 	if(!conf.httprules) conf.httprules = rule;
 	else {
@@ -1495,7 +1598,7 @@ static int h_ace(int argc, unsigned char **argv){
 		tl->ace = acl;
 	
 		if((acl->action == COUNTIN)||(acl->action == COUNTOUT)||(acl->action == COUNTALL)) {
-			unsigned long lim;
+			uint64_t lim = 0;
 
 			tl->comment = ( char *)argv[1];
 			while(isdigit(*tl->comment))tl->comment++;
@@ -1503,9 +1606,9 @@ static int h_ace(int argc, unsigned char **argv){
 			tl->comment = strdup(tl->comment);
 
 			sscanf((char *)argv[1], "%u", &tl->number);
-			sscanf((char *)argv[3], "%lu", &lim);
+			if(sscanf((char *)argv[3], "%"SCNu64"", &lim) != 1) lim = 0;
 			tl->type = getrotate(*argv[2]);
-			tl->traflim64 =  ((uint64_t)lim)*(1024*1024);
+			tl->traflim64 =  lim*(1024*1024);
 			if(!tl->traflim64) {
 				free(tl);
 				freeacl(acl);
@@ -1801,6 +1904,9 @@ int h_server_verify(int argc, unsigned char **argv);
 int h_no_server_verify(int argc, unsigned char **argv);
 int h_client_mode(int argc, unsigned char **argv);
 #endif
+#ifdef WITH_HTTPSRV
+int h_http_content_type(int argc, unsigned char **argv);
+#endif
 #ifdef WITH_TRANSPARENT
 int h_transparent(int argc, unsigned char **argv);
 int h_notransparent(int argc, unsigned char **argv);
@@ -1826,7 +1932,8 @@ struct commands commandhandlers[]={
 #endif
 #ifdef WITH_HTTPSRV
 	{NULL,  "httpsrv", h_proxy, 1, 0},
-	{NULL,  "http", h_http, 4, 5},
+	{NULL,  "http", h_http, 4, 0},
+	{NULL,  "http_content_type", h_http_content_type, 3, 3},
 #endif
 	{NULL,  "dnspr", h_proxy, 1, 0},
 	{NULL,  "internal", h_internal, 2, 2},
@@ -1982,6 +2089,21 @@ int parsestr (unsigned char *str, unsigned char **argm, int nitems, unsigned cha
 			argm[argc] = 0;
 			return argc;
 		case '$':
+			/* Two dollars stand for one. That is how a literal dollar is
+			   written where a file to include would otherwise be read, and
+			   the second one is dropped here as a quote character is. */
+			if(str[1] == '$'){
+				str1 = str;
+				do {
+					*str1 = *(str1 + 1);
+				}while(*(str1++));
+				if(space){
+					argm[argc++] = str;
+					if(argc >= nitems) return argc;
+					space = 0;
+				}
+				break;
+			}
 			if(comment){
 				if(space){
 					argm[argc++] = str;

src/pcre.c

@@ -265,6 +265,9 @@ static FILTER_ACTION pcre_filter_client(void *fo, struct clientparam * param, vo
 	return (res)? CONTINUE:PASS;
 }
 
+/* What a rewritten buffer keeps free for its caller to append to. */
+#define PCRE_HEADROOM 1024
+
 static FILTER_ACTION pcre_filter_buffer(void *fc, struct clientparam *param, unsigned char ** buf_p, int * bufsize_p, int offset, int * length_p){
 	PCRE2_SIZE *ovector;
 	int count = 0;
@@ -324,12 +327,17 @@ static FILTER_ACTION pcre_filter_buffer(void *fc, struct clientparam *param, uns
 			else if(*replace == '$' && isnumber(*(replace+1))){
 				replace ++;
 				num = atoi(replace);
+				/* Past the digits first, and only then decide whether
+				   the group is one to copy: the pass which measured
+				   this string did it in that order, and a reference it
+				   counted as nothing must not be written out as its
+				   own digits here. */
+				while(isnumber(*replace)) replace++;
 				if(num > (count - 1)) continue;
 				if(ovector[(num<<1)] == PCRE2_UNSET) continue;
 				if(ovector[(num<<1) + 1] > (PCRE2_SIZE)*length_p || ovector[(num<<1)] > ovector[(num<<1) + 1]) continue;
 				memcpy(target, *buf_p + ovector[(num<<1)], ovector[(num<<1) + 1] - ovector[(num<<1)]);
 				target += (ovector[(num<<1) + 1] - ovector[(num<<1)]);
-				while(isnumber(*replace)) replace++;
 			}
 			else {
 				*target++ = *replace++;
@@ -338,7 +346,13 @@ static FILTER_ACTION pcre_filter_buffer(void *fc, struct clientparam *param, uns
 		repsz = (int)(target - tmpbuf);
 		memcpy(target, *buf_p + ovector[1], *length_p - ovector[1]);
 		if((ovector[0] + replen + 1) > *bufsize_p){
-			newbuf = pl->mallocfunc(ovector[0] + replen + 1);
+			/* Room beyond what was produced: whoever asked for the
+			   filtering usually has something of its own to add, and a
+			   buffer sized to the last byte written leaves nowhere to
+			   put it. The size reported is the size allocated. */
+			int newsize = ovector[0] + replen + 1 + PCRE_HEADROOM;
+
+			newbuf = pl->mallocfunc(newsize);
 			if(!newbuf){
 				pl->freefunc(tmpbuf);
 				return CONTINUE;
@@ -346,7 +360,7 @@ static FILTER_ACTION pcre_filter_buffer(void *fc, struct clientparam *param, uns
 			memcpy(newbuf, *buf_p, ovector[0]);
 			pl->freefunc(*buf_p);
 			*buf_p = (unsigned char *)newbuf;
-			*bufsize_p = ovector[0] + replen + 1;
+			*bufsize_p = newsize;
 		}
 		memcpy(*buf_p + ovector[0], tmpbuf, replen);
 		pl->freefunc(tmpbuf);
@@ -629,6 +643,71 @@ static struct symbol regexp_symbols[] = {
 };
 
 
+/* Compiling and matching for patterns outside the pcre commands: a host name
+   or a URL in an http rule, an access rule naming a host. They go through the
+   same compile, with whatever pcre_options is set to, so one kind of regular
+   expression is understood everywhere.
+ */
+void * pcre_pattern_compile(const unsigned char *pattern, char *errbuf, int errlen)
+{
+	pcre2_code *re;
+	int errcode;
+	PCRE2_SIZE erroffset;
+
+	re = pcre2_compile((PCRE2_SPTR)pattern, PCRE2_ZERO_TERMINATED, pcre_options,
+		&errcode, &erroffset, NULL);
+	if(!re){
+		if(errbuf && errlen > 0){
+			PCRE2_UCHAR message[256];
+
+			pcre2_get_error_message(errcode, message, sizeof(message));
+			snprintf(errbuf, errlen, "%s at offset %d", (char *)message, (int)erroffset);
+		}
+		return NULL;
+	}
+	return re;
+}
+
+void pcre_pattern_free(void *re)
+{
+	if(re) pcre2_code_free((pcre2_code *)re);
+}
+
+/* Returns the number of captures placed, or 0 when the subject does not
+   match. Element 0 is the whole match. The match data is per call: a rule is
+   matched from several threads at once.
+ */
+int pcre_pattern_match(void *re, const unsigned char *subject, struct capture *caps, int maxcaps)
+{
+	pcre2_match_data *match_data;
+	PCRE2_SIZE *ovector;
+	int count, i, placed = 0;
+
+	if(!re || !subject) return 0;
+	match_data = pcre2_match_data_create_from_pattern((pcre2_code *)re, NULL);
+	if(!match_data) return 0;
+
+	count = pcre2_match((pcre2_code *)re, (PCRE2_SPTR)subject, PCRE2_ZERO_TERMINATED,
+		0, 0, match_data, NULL);
+	if(count > 0){
+		ovector = pcre2_get_ovector_pointer(match_data);
+		if(count > maxcaps) count = maxcaps;
+		for(i = 0; i < count; i++){
+			if(ovector[i*2] == PCRE2_UNSET){
+				caps[i].start = 0;
+				caps[i].len = 0;
+			}
+			else {
+				caps[i].start = (int)ovector[i*2];
+				caps[i].len = (int)(ovector[i*2+1] - ovector[i*2]);
+			}
+		}
+		placed = count;
+	}
+	pcre2_match_data_free(match_data);
+	return placed;
+}
+
 void pcre_install(void){
 
 	struct filter *flt, *tmpflt;

src/proxy.c

@@ -132,6 +132,12 @@ char * proxy_stringtable[] = {
 };
 
 #define LINESIZE 32768
+/* "Content-Length: " plus 20 digits plus CRLF and a NUL, rounded up */
+#define CLHDRSIZE 48
+/* what the headers this proxy adds of its own can come to: a Forwarded or
+   Via with a host name in it, a Connection, a Proxy-support and a
+   Proxy-Authorization carrying an encoded user and password */
+#define HDRRESERVE 2048
 #define BUFSIZE (LINESIZE*2)
 #define FTPBUFSIZE 1536
 
@@ -151,6 +157,20 @@ static int send_st(struct clientparam *param, int idx){
 	return socksend(param, param->clisock, (unsigned char *)proxy_stringtable[idx], pst_len(idx), conf.timeouts[STRING_S]);
 }
 
+/* Makes room in a buffer whose size is tracked. A filter may hand back one
+   holding exactly what it produced, so nothing may be added to it without
+   asking for the room first. Returns 1 when the room cannot be had. */
+static int growbuf(unsigned char **buf, int *bufsize, int need){
+	unsigned char *newbuf;
+
+	if(need <= *bufsize) return 0;
+	need += BUFSIZE;		/* for what follows too, not just this */
+	if(!(newbuf = realloc(*buf, need))) return 1;
+	*buf = newbuf;
+	*bufsize = need;
+	return 0;
+}
+
 static void freeptr(void *p){
 	void **pp = (void **)p;
 	if(*pp) { free(*pp); *pp = NULL; }
@@ -648,6 +668,10 @@ for(;;){
 	RETURN(0);
  }
  if(action != PASS) RETURN(517);
+ /* A filter may have returned a buffer sized to exactly what it produced.
+    The headers this proxy adds of its own go in after it, so the room for
+    them is taken back before anything is written. */
+ if(growbuf(&buf, &bufsize, inbuf + HDRRESERVE)) RETURN(21);
  param->nolongdatfilter = 0;
 
 #endif
@@ -681,6 +705,7 @@ for(;;){
 	contentlength64 = param->cliinbuf;
 	param->nolongdatfilter = 1;
   }
+  if(growbuf(&buf, &bufsize, (int)strlen((char *)buf) + CLHDRSIZE)) RETURN(21);
   sprintf((char*)buf+strlen((char *)buf), "Content-Length: %"PRIu64"\r\n", contentlength64);
  }
 
@@ -1158,6 +1183,7 @@ for(;;){
 	RETURN(0);
  }
  if(action != PASS) RETURN(517);
+ if(growbuf(&buf, &bufsize, inbuf + HDRRESERVE)) RETURN(21);
 
  param->nolongdatfilter = 0;
 
@@ -1181,6 +1207,7 @@ for(;;){
 	}
 	if(action != PASS) RETURN(517);
 	contentlength64 = param->srvinbuf;
+	if(growbuf(&buf, &bufsize, (int)strlen((char *)buf) + CLHDRSIZE)) RETURN(21);
 	sprintf((char*)buf+strlen((char *)buf), "Content-Length: %"PRIu64"\r\n", contentlength64);
 	hascontent = 1;
   }
@@ -1271,6 +1298,16 @@ REQUESTEND:
 	RETURN(0);
  }
  if(param->transparent && (!ckeepalive || !keepalive)) {RETURN (0);}
+ /* Another service read this request and handed it here to be answered. It
+    keeps the connection and decides what the next request on it is, so this
+    one is done. Whatever was opened towards the server stays open in param
+    for the next one. */
+ if(param->onerequest){
+	/* 2 says the client connection may carry another request, 1 that it may
+	   not, which is what the service holding it needs to know. */
+	param->onerequest = (ckeepalive && keepalive)? 2 : 1;
+	RETURN(0);
+ }
  logurl(param, (char *)buf, (char *)req, ftp);
  param->status = 0;
 

src/proxy.h

@@ -169,6 +169,7 @@ void daemonize(void);
 
 #ifndef _WIN32
 size_t threadstacksize(int extra);
+
 #endif
 
 #ifdef WITH_ODBC
@@ -386,7 +387,19 @@ int readconfig(FILE * fp);
 void initcommands(void);
 int connectwithpoll(struct clientparam *param, SOCKET sock, struct sockaddr *sa, SASIZETYPE size, int to);
 int bindwithrange(struct clientparam *param, SOCKET sock, PROXYSOCKADDRTYPE *sa, uint32_t range);
+#ifdef WITH_PCRE
+/* One regular expression implementation for the whole program: the pcre
+   commands and every pattern that carries a pcre: prefix. */
+void * pcre_pattern_compile(const unsigned char *pattern, char *errbuf, int errlen);
+void pcre_pattern_free(void *re);
+int pcre_pattern_match(void *re, const unsigned char *subject, struct capture *caps, int maxcaps);
+#endif
+
+int pushbackcli(struct clientparam * param, const unsigned char * data, int len);
 int parsepattern(struct hostname *h, unsigned char *arg);
+int parsepathpattern(struct hostname *h, unsigned char *arg);
+int patternmatchcaps(const struct hostname *h, const unsigned char *str,
+	struct capture *caps, int *ncaps);
 int patternmatch(const struct hostname *h, const unsigned char *str);
 int patternmatchpos(const struct hostname *h, const unsigned char *str, int *start, int *len);
 void applyportranges(struct clientparam * param, struct ace * acentry);

src/sockgetchar.c

@@ -88,6 +88,35 @@ int sockgetcharcli(struct clientparam * param, int timeosec, int timeousec){
 	return (int)*param->clibuf;
 }
 
+/* Put bytes back in front of whatever the client has not been read yet, so a
+   service which has already taken a request off the socket can hand it to
+   another one, which reads it the way it reads anything else. */
+int pushbackcli(struct clientparam * param, const unsigned char * data, int len){
+	unsigned left = 0;
+	unsigned need;
+
+	if(len <= 0) return 0;
+	if(param->clibuf) left = param->cliinbuf - param->clioffset;
+	need = (unsigned)len + left;
+
+	if(!param->clibuf){
+		if(!(param->clibuf = malloc(need > SRVBUFSIZE? need : SRVBUFSIZE))) return 1;
+		param->clibufsize = need > SRVBUFSIZE? need : SRVBUFSIZE;
+	}
+	else if(param->clibufsize < need){
+		unsigned char *nb = realloc(param->clibuf, need);
+
+		if(!nb) return 1;
+		param->clibuf = nb;
+		param->clibufsize = need;
+	}
+	if(left) memmove(param->clibuf + len, param->clibuf + param->clioffset, left);
+	memcpy(param->clibuf, data, (size_t)len);
+	param->clioffset = 0;
+	param->cliinbuf = need;
+	return 0;
+}
+
 unsigned long sockfillbuffcli(struct clientparam * param, unsigned long size, int timeosec){
 	int len;
 

src/structures.h

@@ -350,21 +350,55 @@ struct period {
 
 #define MATCHBEGIN 1
 #define MATCHEND 2
+/* A pattern is either the star form above, matched by matchtype, or a regular
+   expression compiled once when the configuration is read. */
+#define MATCHGLOB  4	/* stars anywhere: * within a path element, ** across */
+#define MATCHREGEX 5
+
+/* What a star or a capturing group stood for. Element 0 is the whole
+   subject, so a template writes it as $0 and the groups as $1 upwards. */
+#define MAXCAPTURES 10
+struct capture {
+	int start;
+	int len;
+};
 
 struct hostname {
 	struct hostname *next;
 	unsigned char * name;
 	int matchtype;
+	void * re;		/* compiled regular expression, MATCHREGEX only */
 };
 
 /* A request handed to an http operation. */
 struct httpreq {
+	struct capture caps[MAXCAPTURES];
+	int ncaps;
+	struct capture hostcaps[MAXCAPTURES];
+	int nhostcaps;
+	const char *ctype;
+	const char *hdrs;
+	int maxage;
+	int code;
+	time_t ims;		/* what If-Modified-Since asked about, or 0 */
+	int version;		/* 0 for HTTP/1.0, 1 for HTTP/1.1 */
+	int keepalive;		/* whether the connection carries another request */
+	int first;		/* the first request on this connection */
+	int chunkedreq;		/* a body this server does not know how to read */
+	int proxy;		/* the client asked the way it asks a proxy */
+	int connect;		/* and asked for a tunnel */
+	int mayproxy;		/* an access rule sent this to the local proxy */
+	unsigned char *raw;	/* the request as it arrived, for handing on */
+	int rawlen, rawsize;
+	int drained;		/* the body has been read and thrown away */
+	char *lasthost;		/* where the last request on this connection went */
+	void *handoff;		/* a child which takes the connection over */
 	struct clientparam *param;
 	char method[16];
 	char path[256];
 	char query[512];
 	char host[256];
-	unsigned long contentlen;
+	uint64_t contentlen;
 	int globstart, globlen;
 };
 
@@ -377,6 +411,10 @@ struct httprule {
 	struct hostname url;
 	int op;
 	unsigned char *params;
+	unsigned char *ctype;	/* type named by the rule, or NULL to work it out */
+	unsigned char *hdrs;	/* headers the rule adds, already CRLF separated */
+	int maxage;		/* seconds to allow caching for, or -1 to say nothing */
+	int code;		/* status the rule answers with, or 0 for the usual */
 };
 
 struct ace {
@@ -715,6 +753,12 @@ struct clientparam {
 	int udp_nhops;
 	struct ace *lastace;
 	time_t time_start;
+	/* Set by a service which read a request itself and handed it to another
+	   child to answer: that child answers this one request and returns,
+	   leaving the connection to the service which called it. Added last so
+	   that a plugin built against an older header still finds the fields it
+	   knows where they were. */
+	int onerequest;
 };
 
 struct filemon {

tests/cases/auto.py

@@ -21,7 +21,7 @@ def run(t):
         ssl_serv
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{secure}
         ssl_noserv"""
 
@@ -30,7 +30,7 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{plain}
 {tls_origin}
 

tests/cases/httpsrv_auth.py

@@ -6,14 +6,14 @@ def run(t):
     openport = t.free_port()
     t.start("httpsrv_auth", f"""
         log
-        http * /echo echo
+        http echo * /echo
         auth strong
         users alice:CL:secret bob:CL:hunter2
         allow alice
         httpsrv -p{srv}
 
         flush
-        http * /echo echo
+        http echo * /echo
         auth iponly
         allow *
         httpsrv -p{openport}

tests/cases/httpsrv_files.py

@@ -0,0 +1,195 @@
+"""The operations that serve a filesystem: file, cache, redir and rewrite.
+
+A rule maps a request onto a path with a template, where $1 upwards stand for
+what the stars or the groups of a regular expression matched.
+"""
+
+import os
+
+
+def run(t):
+    root = os.path.join(t.tmpdir, "web")
+    os.makedirs(os.path.join(root, "picts", "set"), exist_ok=True)
+    with open(os.path.join(root, "a.html"), "w") as fp:
+        fp.write("<h1>hello</h1>")
+    with open(os.path.join(root, "big.bin"), "wb") as fp:
+        fp.write(b"x" * 300000)          # past a single send, and past the cache limit
+    with open(os.path.join(root, "picts", "set", "dog.gif"), "wb") as fp:
+        fp.write(b"GIF89a-pretend")
+
+    with open(os.path.join(root, "b.webp"), "wb") as fp:
+        fp.write(b"RIFF-pretend")
+
+    port = t.free_port()
+    t.start("httpsrv_files", f"""
+        log
+        auth iponly
+        allow *
+        http rewrite * /alias/** "/w/$1"
+        http file  * /w/*.html "{root}/$1.html"
+        http file  * /big       {root}/big.bin
+        http cache * /c/*.html  "{root}/$1.html"
+        http cache * "pcre:^/(.*)/pic/(.*)\\.(gif|jpeg)$" "{root}/picts/$1/$2.$3"
+        http redir * /old/**    301 "https://example.org/$1"
+        http redir * /moved     /w/a.html
+        http file  * /rel/*.html "web/$1.html"
+        http echo  * /echo
+
+        http_content_type .webp image/webp
+        http_content_type dat   application/x-mydata
+        http file * /ct/*.webp  "{root}/$1.webp"
+        http file * /named/*.html "{root}/$1.html" text/x-named
+        http file * /star/*.html  "{root}/$1.html" *
+        http cache * /ctc/*.webp  "{root}/$1.webp"
+
+        http file  * /aged/*.html  "{root}/$1.html" * 3600
+        http cache * /aged2/*.html "{root}/$1.html" * 60
+        http file  * /extra/*.html "{root}/$1.html" * * "X-One: 1\\nX-Two: two words"
+        http file  * /err/*.html   "{root}/$1.html" * * "X-Served: static" 404
+        http reply * /ok**
+        http reply * /nobody**  204
+        http reply * /down**    503 "Retry-After: 30"
+        http cache * /held/*.html "{root}/$1.html" * 30
+        httpsrv -p{port}
+    """, ports=[port])
+
+    url = f"http://127.0.0.1:{port}"
+
+    # --- file -------------------------------------------------------------
+    r = t.http(url + "/w/a.html")
+    t.eq(200, r.status, "a file is served")
+    t.contains(r, "<h1>hello</h1>", "with its content")
+    t.eq("text/html", r.header("Content-Type"), "and a type taken from the name")
+    t.eq(str(len("<h1>hello</h1>")), r.header("Content-Length"), "and its length")
+
+    t.eq(404, t.http(url + "/w/nosuch.html").status, "a missing file is not found")
+    big = t.http(url + "/big")
+    t.eq(300000, big.length, "a large file arrives whole")
+    t.eq("300000", big.header("Content-Length"), "and is announced by its length")
+    t.eq(None, big.header("Transfer-Encoding"),
+         "a file is never sent chunked")
+    t.eq("300000", t.http(url + "/big", method="HEAD").header("Content-Length"),
+         "HEAD gives the length without the body")
+    t.eq(200, t.http(url + "/w/a.html", method="HEAD").status, "HEAD is answered")
+    t.eq(0, t.http(url + "/w/a.html", method="HEAD").length, "HEAD carries no body")
+
+    # --- cache ------------------------------------------------------------
+    first = t.http(url + "/c/a.html")
+    second = t.http(url + "/c/a.html")
+    t.eq(200, first.status, "a cached file is served")
+    t.eq(first.text, second.text, "and the same on the next request")
+    t.contains(second, "<h1>hello</h1>", "from memory this time")
+
+    # a file changed on disk is noticed rather than served from before
+    with open(os.path.join(root, "a.html"), "w") as fp:
+        fp.write("<h1>changed</h1>")
+    t.contains(t.http(url + "/c/a.html"), "changed",
+               "a file replaced on disk is read again")
+
+    # --- what the stars stand for -----------------------------------------
+    r = t.http(url + "/set/pic/dog.gif")
+    t.eq(200, r.status, "a regular expression maps a request onto a path")
+    t.contains(r, "GIF89a", "and the file is served")
+    t.eq("image/gif", r.header("Content-Type"), "with the type of that name")
+
+    # --- redir ------------------------------------------------------------
+    r = t.http(url + "/old/thing")
+    t.eq(301, r.status, "a redirect uses the status it was given")
+    t.eq("https://example.org/thing", r.header("Location"),
+         "and a location built from the request")
+    t.eq(302, t.http(url + "/moved").status, "without a status it is 302")
+
+    # --- rewrite ----------------------------------------------------------
+    r = t.http(url + "/alias/a.html")
+    t.eq(200, r.status, "a rewritten request reaches the rule after it")
+    t.contains(r, "changed", "and is served from the path it was rewritten to")
+
+    # --- the type a reply carries -------------------------------------------
+    # Worked out from the name, using what the configuration has registered
+    # on top of what is built in, unless the rule says otherwise.
+    t.eq("image/webp", t.http(url + "/ct/b.webp").header("Content-Type"),
+         "a registered extension names the type")
+    t.eq("image/webp", t.http(url + "/ctc/b.webp").header("Content-Type"),
+         "and a cached file is answered the same way")
+    t.eq("text/x-named", t.http(url + "/named/a.html").header("Content-Type"),
+         "a rule may name the type itself")
+    t.eq("text/html", t.http(url + "/star/a.html").header("Content-Type"),
+         "and a star there leaves it to the name of the file")
+
+    # --- what a rule adds to the answer ------------------------------------
+    r = t.http(url + "/aged/a.html")
+    t.eq("max-age=3600", r.header("Cache-Control"), "a rule may describe caching")
+    t.eq("max-age=60", t.http(url + "/aged2/a.html").header("Cache-Control"),
+         "a cached file is answered the same way")
+    t.eq(None, t.http(url + "/w/a.html").header("Cache-Control"),
+         "and a rule which says nothing sends nothing")
+
+    r = t.http(url + "/extra/a.html")
+    t.eq("1", r.header("X-One"), "a rule may add headers")
+    t.eq("two words", r.header("X-Two"),
+         "the second of them arrives whole, spaces and all")
+
+    r = t.http(url + "/err/a.html")
+    t.eq(404, r.status, "a rule may answer with the status it names")
+    t.contains(r, "<h1>", "and the file is still the body")
+    t.eq("static", r.header("X-Served"),
+         "what the rule adds goes with the status the rule asked for")
+
+    # a refusal the server decided on is its own answer
+    r = t.http(url + "/err/nosuch.html")
+    t.eq(404, r.status, "a missing file is still not found")
+    t.eq(None, r.header("X-Served"), "and carries none of the rule's headers")
+    t.eq(None, t.http(url + "/aged/nosuch.html").header("Cache-Control"),
+         "nor what it said about caching")
+
+    # --- reply --------------------------------------------------------------
+    r = t.http(url + "/ok")
+    t.eq(200, r.status, "reply answers with 200 by default")
+    t.eq("0", r.header("Content-Length"), "with a length of zero")
+    t.eq(0, r.length, "and no body")
+
+    r = t.http(url + "/nobody")
+    t.eq(204, r.status, "reply answers with the status it was given")
+    t.eq(None, r.header("Content-Length"),
+         "and a status carrying no body is sent without a length")
+
+    r = t.http(url + "/down")
+    t.eq(503, r.status, "reply serves a refusal the configuration decided on")
+    t.eq("30", r.header("Retry-After"), "with the headers that go with it")
+
+    # --- a client which has the file already --------------------------------
+    r = t.http(url + "/w/a.html")
+    stamp = r.header("Last-Modified")
+    t.ne(None, stamp, "a file is answered with the time it was last changed")
+
+    r = t.http(url + "/w/a.html", headers={"If-Modified-Since": stamp})
+    t.eq(304, r.status, "and an unchanged file is answered 304")
+    t.eq(0, r.length, "which carries no body")
+    t.eq(None, r.header("Content-Length"), "and no length")
+    t.eq(stamp, r.header("Last-Modified"), "but still says when the file changed")
+
+    t.eq(200, t.http(url + "/w/a.html",
+                     headers={"If-Modified-Since": "Sun, 06 Nov 1994 08:49:37 GMT"}).status,
+         "an older date is answered with the file")
+    t.eq(200, t.http(url + "/w/a.html",
+                     headers={"If-Modified-Since": "not a date at all"}).status,
+         "and a date which cannot be read is treated as none")
+    t.eq(304, t.http(url + "/c/a.html", headers={"If-Modified-Since": stamp}).status,
+         "a file answered from memory is conditional in the same way")
+    t.eq(404, t.http(url + "/err/a.html", headers={"If-Modified-Since": stamp}).status,
+         "a rule with a status of its own is not turned into a 304")
+
+    # --- a rule which says how long its copy may be held --------------------
+    t.contains(t.http(url + "/held/a.html"), "changed", "a held file is served")
+    with open(os.path.join(root, "a.html"), "w") as fp:
+        fp.write("<h1>replaced</h1>")
+    t.not_contains(t.http(url + "/held/a.html"), "replaced",
+                   "and within its max-age the disk is not looked at again")
+    t.contains(t.http(url + "/c/a.html"), "replaced",
+               "while a rule without one notices the change at once")
+
+    # --- the paths a rule may not build ------------------------------------
+    t.eq(403, t.http(url + "/rel/a.html").status,
+         "a relative target is refused")
+    t.ne(200, t.http(url + "/w/../etc/passwd").status,
+         "a request climbing out of the tree is refused")

tests/cases/httpsrv_keepalive.py

@@ -0,0 +1,109 @@
+"""Keep-alive: which answers may be followed by another request.
+
+The next request begins where the last answer ended, so a connection is only
+kept when the length of what was sent is known exactly and the body of the
+request was read to its end. Everything else closes, which is the safe way to
+be wrong.
+"""
+
+import os
+
+
+def run(t):
+    root = os.path.join(t.tmpdir, "ka")
+    os.makedirs(root, exist_ok=True)
+    with open(os.path.join(root, "a.html"), "w") as fp:
+        fp.write("<h1>hello</h1>")
+
+    port = t.free_port()
+    t.start("httpsrv_keepalive", f"""
+        log
+        auth iponly
+        allow *
+        http file  * /w/*.html "{root}/$1.html"
+        http reply * /ok**  200
+        http echo  * /echo**
+        http data  * /chunked** size=100&chunked=1
+        httpsrv -p{port}
+    """, ports=[port])
+
+    def req(path, version="1.1", extra="", body=""):
+        head = (f"GET {path} HTTP/{version}\r\nHost: t\r\n{extra}\r\n")
+        if body:
+            head = head.replace("GET", "POST", 1)
+        return head + body
+
+    def session(*requests, quiet=0.5):
+        text, closed = t.raw_session(port, "".join(requests), quiet=quiet)
+        return text, closed, text.count("HTTP/1.")
+
+    # --- what keeps the connection ---------------------------------------
+    text, closed, n = session(req("/w/a.html"), req("/ok"),
+                              req("/w/a.html", extra="Connection: close\r\n"))
+    t.eq(3, n, "three 1.1 requests are answered on one connection")
+    t.eq(True, closed, "and the one asking to close ends it")
+    t.contains(text, "Connection: keep-alive", "the answers say the connection is kept")
+    t.eq(2, text.count("<h1>hello</h1>"), "each file arrives whole")
+
+    text, closed, n = session(req("/w/a.html", version="1.0"), req("/ok", version="1.0"))
+    t.eq(1, n, "a 1.0 request without the header is answered once")
+    t.eq(True, closed, "and the connection ends")
+    t.contains(text, "Connection: close", "which the answer says")
+
+    text, closed, n = session(req("/w/a.html", version="1.0",
+                                  extra="Connection: keep-alive\r\n"),
+                              req("/ok", version="1.0",
+                                  extra="Connection: close\r\n"))
+    t.eq(2, n, "a 1.0 client asking for keep-alive gets it")
+
+    # a request carrying a body: the next one begins after it
+    text, closed, n = session(req("/echo", extra="Content-Length: 5\r\n", body="hello"),
+                              req("/ok", extra="Connection: close\r\n"))
+    t.eq(2, n, "a body which was read to its end leaves the stream in place")
+    t.contains(text, "content.length=5", "and the body was seen")
+
+    # --- what ends it -----------------------------------------------------
+    text, closed, n = session(req("/echo", extra="Transfer-Encoding: chunked\r\n"),
+                              req("/ok"))
+    t.eq(1, n, "a request body this server cannot frame ends the connection")
+    t.eq(True, closed, "the connection is closed rather than left mid-body")
+
+    # A body longer than the server is willing to read leaves the rest of it
+    # in the stream, so the connection cannot carry another request. The send
+    # may not even finish - the server answers and closes part way through -
+    # which is the same answer from the other side.
+    big = "x" * 1500000
+    text, closed, n = session(req("/echo", extra="Content-Length: 1500000\r\n", body=big),
+                              quiet=2)
+    t.eq(1, n, "a body past what the server will read is answered once")
+    t.contains(text, "Connection: close",
+               "and the answer ends the connection rather than leaving the rest to be read")
+
+    # --- answers of other shapes -----------------------------------------
+    text, closed, n = session(req("/chunked"), req("/ok", extra="Connection: close\r\n"))
+    t.eq(2, n, "a chunked answer may be followed by another request")
+
+    text, closed, n = session(req("/chunked", version="1.0"), req("/ok", version="1.0"))
+    t.eq(1, n, "but not for a client which has no chunked encoding to read")
+
+    stamp = t.http(f"http://127.0.0.1:{port}/w/a.html").header("Last-Modified")
+    text, closed, n = session(req("/w/a.html", extra=f"If-Modified-Since: {stamp}\r\n"),
+                              req("/ok", extra="Connection: close\r\n"))
+    t.eq(2, n, "a 304 carries no body and the next request follows it")
+    t.contains(text, "304", "and it is a 304")
+
+    # --- the administration pages always close ---------------------------
+    aport = t.free_port()
+    t.start("httpsrv_keepalive_admin", f"""
+        log
+        auth iponly
+        allow *
+        http file * /w/*.html "{root}/$1.html"
+        admin -p{aport}
+    """, ports=[aport])
+
+    text, closed = t.raw_session(aport,
+        f"GET /w/a.html HTTP/1.1\r\nHost: t\r\n\r\nGET /C HTTP/1.1\r\nHost: t\r\n\r\n"
+        f"GET /w/a.html HTTP/1.1\r\nHost: t\r\n\r\n")
+    t.eq(2, text.count("HTTP/1."), "an administration page is the last thing on a connection")
+    t.eq(True, closed, "which the server closes, since the page states no length")

tests/cases/httpsrv_ops.py

@@ -9,9 +9,9 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
-        http * /data data
+        http data * /data
-        http * /small data size=64
+        http data * /small size=64
         httpsrv -p{srv}
     """, ports=[srv])
 

tests/cases/httpsrv_parsing.py

@@ -11,8 +11,8 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
-        http * /safe/* echo
+        http echo * /safe/**
         httpsrv -p{srv}
     """, ports=[srv])
 
@@ -62,3 +62,28 @@ def run(t):
                headers={"Content-Type": "application/x-www-form-urlencoded"})
     t.contains(r, "method=POST", "POST reaches the handler")
     t.contains(r, "content.length=9", "the POST content length is parsed")
+
+    # --- dollars in the configuration ------------------------------------
+    # Outside quotes a dollar begins the name of a file to include, so an
+    # argument holding one is quoted. Two dollars stand for one, which is how
+    # a dollar reaches a rule as text.
+    dsrv = t.free_port()
+    t.start("httpsrv_dollar", f"""
+        log
+        auth iponly
+        allow *
+        http redir * /old** 301 "http://example.org/x$$y/$1"
+        http redir * "pcre:^/re/([a-z]+)$" 302 "http://example.org/re/$1"
+        http echo * /**
+        httpsrv -p{dsrv}
+    """, ports=[dsrv])
+
+    durl = f"http://127.0.0.1:{dsrv}"
+    r = t.http(durl + "/old/a")
+    t.eq(301, r.status, "a rule holding a doubled dollar loads")
+    t.eq("http://example.org/x$y//a", r.header("Location"),
+         "and two dollars reach the location as one")
+    t.eq(302, t.http(durl + "/re/abc").status,
+         "a quoted regular expression keeps its anchor")
+    t.eq(200, t.http(durl + "/re/ab9").status,
+         "and the anchor is real: what it excludes falls through")

tests/cases/httpsrv_proxypass.py

@@ -0,0 +1,203 @@
+"""A service which is both a site and a proxy.
+
+The rules answer what they have; anything else is handed to the proxy code,
+which authenticates as a proxy and fetches it. The same connection carries
+both kinds of request.
+"""
+
+import os
+
+
+def run(t):
+    root = os.path.join(t.tmpdir, "pp")
+    os.makedirs(root, exist_ok=True)
+    with open(os.path.join(root, "a.html"), "w") as fp:
+        fp.write("<h1>local</h1>")
+
+    # two origins, so a change of destination is visible
+    one = t.free_port()
+    two = t.free_port()
+    t.start("httpsrv_proxypass_origins", f"""
+        auth iponly
+        allow *
+        http echo * /**
+        httpsrv -p{one}
+
+        flush
+        auth iponly
+        allow *
+        http echo * /**
+        httpsrv -p{two}
+    """, ports=[one, two])
+
+    # --- the rule which hands a request on ---------------------------------
+    srv = t.free_port()
+    t.start("httpsrv_proxypass", f"""
+        log
+        auth iponly
+        allow *
+        http file * /local/*.html "{root}/$1.html"
+        http reply * /health** 200
+        http proxypass * /**
+        httpsrv -p{srv}
+    """, ports=[srv])
+
+    url = f"http://127.0.0.1:{srv}"
+    t.contains(t.http(url + "/local/a.html"), "<h1>local</h1>",
+               "a rule of its own is still answered here")
+    t.eq(200, t.http(url + "/health").status, "and so is another")
+
+    r = t.http(f"http://127.0.0.1:{one}/echo", proxy=f"127.0.0.1:{srv}")
+    t.eq(200, r.status, "a request the rules do not answer is proxied")
+    t.contains(r, "path=/echo", "and the origin sees it")
+
+    # a client which sends an origin-form request with a Host header reaches
+    # the same place: what decides is which rule matches, not the form
+    r = t.http(url + "/echo", headers={"Host": f"127.0.0.1:{one}"})
+    t.contains(r, "path=/echo", "an origin-form request is proxied the same way")
+
+    # --- an access rule which sends the rest to the proxy -------------------
+    # allow, with a chain to the local proxy, then a second rule for the pass
+    # the proxy itself makes
+    rsrv = t.free_port()
+    t.start("httpsrv_proxypass_acl", f"""
+        log
+        auth iponly
+        allow *
+        parent 1000 http 0.0.0.0 0
+        allow *
+        http file * /local/*.html "{root}/$1.html"
+        httpsrv -p{rsrv}
+    """, ports=[rsrv])
+
+    rurl = f"http://127.0.0.1:{rsrv}"
+    t.contains(t.http(rurl + "/local/a.html"), "<h1>local</h1>",
+               "a rule still wins over the redirect")
+    r = t.http(f"http://127.0.0.1:{one}/echo", proxy=f"127.0.0.1:{rsrv}")
+    t.eq(200, r.status, "and what no rule matches goes to the proxy the rule named")
+
+    # --- rules after the chain decide what the proxy may fetch -------------
+    # The service answers for itself on the first pass, so an address or a
+    # port there is the one the client connected to; on the pass the proxy
+    # makes, it is the one the request names.
+    gsrv = t.free_port()
+    t.start("httpsrv_proxypass_gate", f"""
+        log
+        auth iponly
+        allow *
+        parent 1000 http 0.0.0.0 0
+        allow * * 127.0.0.1/32 {one}
+        deny *
+        httpsrv -p{gsrv}
+    """, ports=[gsrv])
+
+    t.eq(200, t.http(f"http://127.0.0.1:{one}/echo", proxy=f"127.0.0.1:{gsrv}").status,
+         "a destination a later rule allows is fetched")
+    t.eq(403, t.http(f"http://127.0.0.1:{two}/echo", proxy=f"127.0.0.1:{gsrv}").status,
+         "and one no rule allows is refused")
+
+    # a deny written before the rule carrying the chain applies as well
+    bsrv = t.free_port()
+    t.start("httpsrv_proxypass_deny", f"""
+        log
+        auth iponly
+        deny * * 127.0.0.1/32 {two}
+        allow *
+        parent 1000 http 0.0.0.0 0
+        allow *
+        httpsrv -p{bsrv}
+    """, ports=[bsrv])
+
+    t.eq(200, t.http(f"http://127.0.0.1:{one}/echo", proxy=f"127.0.0.1:{bsrv}").status,
+         "what the deny does not name is still fetched")
+    t.eq(403, t.http(f"http://127.0.0.1:{two}/echo", proxy=f"127.0.0.1:{bsrv}").status,
+         "a deny before the chain stops the request too")
+
+    # --- one connection, both kinds of request -----------------------------
+    text, closed = t.raw_session(srv,
+        f"GET /local/a.html HTTP/1.1\r\nHost: t\r\n\r\n"
+        f"GET http://127.0.0.1:{one}/echo HTTP/1.1\r\nHost: 127.0.0.1:{one}\r\n\r\n"
+        f"GET http://127.0.0.1:{two}/echo HTTP/1.1\r\nHost: 127.0.0.1:{two}\r\n\r\n"
+        f"GET /local/a.html HTTP/1.1\r\nHost: t\r\nConnection: close\r\n\r\n",
+        quiet=2)
+    t.eq(4, text.count("HTTP/1."), "four requests are answered on one connection")
+    t.eq(2, text.count("<h1>local</h1>"), "two of them here")
+    t.eq(2, text.count("peer.addr="), "and two by the origins")
+    t.eq(True, closed, "the last one ends it")
+
+    # --- every kind of rule on the same connection --------------------------
+    with open(os.path.join(root, "f.html"), "w") as fp:
+        fp.write("FILEBODY")
+    with open(os.path.join(root, "c.html"), "w") as fp:
+        fp.write("CACHEBODY")
+
+    msrv = t.free_port()
+    t.start("httpsrv_proxypass_mix", f"""
+        log
+        auth iponly
+        allow *
+        http file  * /f/*.html "{root}/$1.html"
+        http cache * /c/*.html "{root}/$1.html"
+        http proxypass * /**
+        httpsrv -p{msrv}
+    """, ports=[msrv])
+
+    proxied = f"GET http://127.0.0.1:{one}/echo HTTP/1.1\r\nHost: 127.0.0.1:{one}\r\n\r\n"
+    text, closed = t.raw_session(msrv,
+        "GET /f/f.html HTTP/1.1\r\nHost: t\r\n\r\n"
+        "GET /c/c.html HTTP/1.1\r\nHost: t\r\n\r\n"
+        + proxied +
+        "GET /c/c.html HTTP/1.1\r\nHost: t\r\n\r\n"
+        + proxied +
+        "GET /f/f.html HTTP/1.1\r\nHost: t\r\nConnection: close\r\n\r\n",
+        quiet=2)
+    t.eq(6, text.count("HTTP/1."), "file, cache and proxypass share one connection")
+    t.eq(2, text.count("FILEBODY"), "both files arrive")
+    t.eq(2, text.count("CACHEBODY"), "both cached files arrive")
+    t.eq(2, text.count("peer.addr="), "and both proxied requests arrive")
+    t.eq(True, closed, "the request asking to close ends it")
+
+    # --- a proxied answer of unstated length ends the connection ------------
+    # Its body is delimited by the close, so nothing can follow it here
+    # either: the client has to ask again on a new connection.
+    closer = t.free_port()
+    stop = t.raw_server(closer,
+                        b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nCLOSEDELIMITED",
+                        close_after=True)
+    try:
+        text, closed = t.raw_session(msrv,
+            f"GET http://127.0.0.1:{closer}/x HTTP/1.1\r\nHost: 127.0.0.1:{closer}\r\n\r\n"
+            "GET /f/f.html HTTP/1.1\r\nHost: t\r\n\r\n", quiet=2)
+        t.eq(1, text.count("HTTP/1."), "the answer of unstated length is the last one")
+        t.contains(text, "CLOSEDELIMITED", "and its body still arrives whole")
+        t.eq(True, closed, "the connection ends with it")
+    finally:
+        stop()
+
+    # --- credentials go where a proxy expects them --------------------------
+    asrv = t.free_port()
+    t.start("httpsrv_proxypass_auth", f"""
+        log
+        users u:CL:p
+        auth strong
+        allow u
+        http file * /local/*.html "{root}/$1.html"
+        http proxypass * /**
+        httpsrv -p{asrv}
+    """, ports=[asrv])
+
+    aurl = f"http://127.0.0.1:{asrv}"
+    r = t.http(f"http://127.0.0.1:{one}/echo", proxy=f"127.0.0.1:{asrv}")
+    t.eq(407, r.status, "a proxy-style request with no credentials is asked for them")
+    t.contains(r.header("Proxy-Authenticate") or "", "Basic",
+               "with the header a proxy client reads")
+
+    r = t.http(f"http://127.0.0.1:{one}/echo", proxy=f"127.0.0.1:{asrv}",
+               proxy_auth=("u", "p"))
+    t.eq(200, r.status, "and is served once they are given")
+
+    r = t.http(aurl + "/local/a.html")
+    t.eq(401, r.status, "a request to the site itself is asked the site's way")
+    t.contains(r.header("WWW-Authenticate") or "", "Basic", "with its own header")
+    t.contains(t.http(aurl + "/local/a.html", auth=("u", "p")), "<h1>local</h1>",
+               "and answered once they are given")

tests/cases/httpsrv_rules.py

@@ -8,19 +8,25 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /exact echo
+        http echo * /exact
-        http * /pre* echo
+        http echo * /pre*
-        http * *.suffix echo
+        http echo * /deep/**
-        http * *mid* echo
+        http echo * **.suffix
-        http host.example.com /byhost echo
+        http echo * **mid**
-        http *.wild.example.com /bywild echo
+        http echo host.example.com /byhost
-        http * /only-first echo
+        http echo *.wild.example.com /bywild
+        http echo * /only-first
+
+        http rewrite_host *.old.example ** "$1.new.example"
+        http rewrite_host "pcre:^legacy-(.*)$" ** "$1.new.example"
+        http rewrite_host * /badhost** "not a host name"
+        http echo one.new.example /**
         httpsrv -p{srv}
 
         flush
         auth iponly
         allow *
-        http * /only-second echo
+        http echo * /only-second
         httpsrv -p{srv2}
     """, ports=[srv, srv2])
 
@@ -31,8 +37,15 @@ def run(t):
     t.eq(404, t.http(url + "/exactly").status,
          "an exact URL does not match a longer path")
     t.eq(200, t.http(url + "/pre").status, "a prefix matches the bare prefix")
-    t.eq(200, t.http(url + "/pretty/deep").status,
+    t.eq(200, t.http(url + "/pretty").status,
-         "a prefix matches a longer path")
+         "a prefix matches a longer name in the same path element")
+
+    # a single star stays inside one element of the path, which is what keeps
+    # a rule from reaching into directories it did not name
+    t.eq(404, t.http(url + "/pretty/deep").status,
+         "a prefix does not cross a slash")
+    t.eq(200, t.http(url + "/deep/a/b/c").status,
+         "a double star does cross one")
     t.eq(200, t.http(url + "/any.suffix").status, "a suffix matches")
     t.eq(404, t.http(url + "/any.suffixx").status,
          "a suffix is anchored at the end")
@@ -67,3 +80,19 @@ def run(t):
          "the second service has its own rules")
     t.eq(404, t.http(f"http://127.0.0.1:{srv2}/only-first").status,
          "the second service does not have the earlier rules")
+
+    # --- a rule which changes the host --------------------------------
+    # The stars of the host pattern are what $1 upwards stand for here, the
+    # way the stars of the URL stand for themselves in a rewrite.
+    r = t.http(url + "/anything", headers={"Host": "one.old.example"})
+    t.eq(200, r.status, "a rewritten host reaches the rules after it")
+    t.contains(r, "host=one.new.example", "and the request carries the new name")
+
+    t.eq(200, t.http(url + "/anything", headers={"Host": "legacy-one"}).status,
+         "a regular expression names the part to keep")
+
+    t.eq(404, t.http(url + "/anything", headers={"Host": "other.example"}).status,
+         "a host no rule rewrites is left as it was")
+
+    t.eq(403, t.http(url + "/badhost", headers={"Host": "x"}).status,
+         "a rule may not build something which is not a host name")

tests/cases/ipv6.py

@@ -21,8 +21,8 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
-        http * /data data
+        http data * /data
         httpsrv -p{origin} -i::1
 
         # reached over IPv6, and allowed to reach IPv6
@@ -102,7 +102,7 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{v4origin}
         {"".join(sections)}
     """, ports=[v4origin] + list(family_ports.values()))

tests/cases/parent_ports.py

@@ -43,7 +43,7 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{srv}
 
         # every outgoing connection binds inside the range

tests/cases/pcre.py

@@ -29,9 +29,9 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
-        http * /secret* echo
+        http echo * /secret**
-        http * /data data
+        http data * /data
         httpsrv -p{origin}
     """, ports=[origin])
 
@@ -88,6 +88,27 @@ def run(t):
     t.eq(200, t.http(url + "/echo", proxy=p).status,
          "an extension that matches nothing changes nothing")
 
+    # --- a regular expression where a host name is expected -----------------
+    # The same prefix works in an access rule and in an http rule, so one
+    # kind of expression is understood wherever a name can be written.
+    named = t.free_port()
+    t.start("pcre_named", f"""
+        log
+        flush
+        nserver 127.0.0.1
+        nscache 1024
+        nsrecord host1.test 127.0.0.1
+        nsrecord other.test 127.0.0.1
+        auth iponly
+        allow * * "pcre:^host[0-9]+\\.test$"
+        proxy -p{named}
+    """, ports=[named])
+
+    t.eq(200, t.http(f"http://host1.test:{origin}/echo", proxy=f"127.0.0.1:{named}").status,
+         "a destination matching the expression is allowed")
+    t.ne(200, t.http(f"http://other.test:{origin}/echo", proxy=f"127.0.0.1:{named}").status,
+         "one that does not match is refused")
+
     # --- rewriting the reply ------------------------------------------------
     p = proxy_with("rewrite_srv",
                    'pcre_rewrite srvheader dunno "text/plain" "text/rewritten"',
@@ -143,7 +164,7 @@ def run(t):
         flush
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{elsewhere}
     """, ports=[elsewhere])
 
@@ -172,3 +193,43 @@ def run(t):
     r = t.http(url + "/echo/old", proxy=p)
     t.eq(200, r.status, "a rewritten request through a parent arrives")
     t.contains(r, "path=/echo/new", "the origin sees the rewritten path through a parent")
+
+    # --- a rewrite which grows the headers ----------------------------------
+    # GHSA-h845-prxq-ww3q: a rewrite that doubles the client headers used to
+    # leave a buffer holding exactly what it produced, and the Content-Length
+    # the data filter regenerates was then written past the end of it.
+    # The origin here reads whatever it is sent and answers the same way every
+    # time: what is being tested is the proxy in the middle, not what a server
+    # is willing to accept in one request.
+    grown = t.free_port()
+    stop = t.raw_server(grown, b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok",
+                        drain=True)
+    try:
+        p = proxy_with("rewrite_grow",
+                       'pcre_rewrite cliheader dunno "(?s).*" "$0$0"',
+                       'pcre clidata dunno *')
+        big = "".join("X-%d: %s\r\n" % (i, chr(65 + i) * 20000) for i in range(5))
+        reply = t.raw_proxy_request(p, f"http://127.0.0.1:{grown}/x",
+                                    extra=big, body="z")
+        t.contains(reply, "200", "a doubled header block with a body is answered")
+        t.contains(t.raw_proxy_request(p, f"http://127.0.0.1:{grown}/x"), "200",
+                   "and the proxy is still there afterwards")
+    finally:
+        stop()
+
+    # A reference to a group the pattern does not have is dropped, and dropped
+    # by both the pass which measures the result and the pass which writes it.
+    p = proxy_with("rewrite_nogroup",
+                   'pcre_rewrite cliheader dunno "(?s)Host:" "$9$9$9$9$9$9$9$9"')
+    r = t.http(url + "/echo", proxy=p, headers={"X-Pad": "P" * 2000})
+    t.eq(200, r.status, "a reference to a group which did not match is left out")
+    t.contains(t.http(url + "/echo", proxy=p), "path=/echo",
+               "and that proxy is still there too")
+
+    # an optional group which took part on one request and not on the next
+    p = proxy_with("rewrite_optgroup",
+                   'pcre_rewrite cliheader dunno "X-Mark: (a)?(b)" "[$1][$2]"')
+    t.eq(200, t.http(url + "/echo", proxy=p, headers={"X-Mark": "ab"}).status,
+         "a group which matched is put in")
+    t.eq(200, t.http(url + "/echo", proxy=p, headers={"X-Mark": "b"}).status,
+         "and one which did not is left out")

tests/cases/portmap.py

@@ -11,8 +11,8 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
-        http * /data data
+        http data * /data
         httpsrv -p{origin}
 
         flush

tests/cases/proxy_http.py

@@ -16,15 +16,15 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
-        http * /data data
+        http data * /data
         httpsrv -p{srv}
 
         # a second origin, used as a destination the rules must keep out
         flush
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{other}
 
         # an open proxy

tests/cases/socks.py

@@ -10,8 +10,8 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
-        http * /data data
+        http data * /data
         httpsrv -p{srv}
 
         flush

tests/cases/ssl.py

@@ -32,7 +32,7 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{origin}
 
         flush
@@ -76,8 +76,8 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
-        http * /data data
+        http data * /data
         httpsrv -p{origin}
 
         flush
@@ -129,7 +129,7 @@ def run(t):
         ssl_serv
         auth iponly
         allow *
-        http * /secret* echo
+        http echo * /secret**
         httpsrv -p{origin}
     """, ports=[origin])
 

tests/cases/tlspr.py

@@ -17,7 +17,7 @@ def run(t):
         ssl_serv
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{origin}
 
         flush

tests/cases/transparent.py

@@ -97,14 +97,14 @@ def run(t):
         log
         auth iponly
         allow *
-        http * /echo* echo
+        http echo * /echo**
         httpsrv -p{origin_port} -i{ORIGIN_ADDR}
 
         # a second server, to tell apart where a connection actually went
         flush
         auth iponly
         allow *
-        http * * data size=13
+        http data * * size=13
         httpsrv -p{decoy_port} -i{DECOY_ADDR}
 
         # a port mapper aimed at the decoy: with the destination taken from

tests/harness.py

@@ -123,6 +123,7 @@ class Tester:
         self.servers = []
         self.checks = []
         self.timeout = 10
+        self._raw_kept = []
         self._skipped = 0
         self._certs = None
         self.logs = []
@@ -321,6 +322,108 @@ class Tester:
         except OSError as exc:
             return f"<no reply: {exc}>"
 
+    def raw_session(self, port, request, host="127.0.0.1", quiet=0.5):
+        """Send bytes and read until the server closes or goes quiet.
+
+        Returns (text, closed). closed says the server ended the connection
+        rather than leaving it open for another request, which is the whole
+        question a keep-alive test asks.
+        """
+        if not isinstance(request, bytes):
+            request = request.encode("latin-1")
+        closed = False
+        chunks = []
+        try:
+            with socket.create_connection((host, port), self.timeout) as sock:
+                try:
+                    sock.sendall(request)
+                except OSError:
+                    # the server answered and closed before taking all of it,
+                    # which is an answer in itself
+                    closed = True
+                sock.settimeout(quiet)
+                while True:
+                    try:
+                        piece = sock.recv(65536)
+                    except socket.timeout:
+                        break               # quiet: the connection is still open
+                    except OSError:
+                        closed = True       # reset: it is not
+                        break
+                    if not piece:
+                        closed = True
+                        break
+                    chunks.append(piece)
+        except OSError as exc:
+            return f"<no reply: {exc}>", True
+        return b"".join(chunks).decode("utf-8", "replace"), closed
+
+    def raw_proxy_request(self, proxy, url, extra="", body="", method=None):
+        """Send one absolute-URI request through a proxy, headers and all.
+
+        For the requests a client library will not send: an oversized header
+        block, or one whose exact bytes matter.
+        """
+        phost, pport = self._hostport(proxy)
+        host, port, path = self._split(url)
+        method = method or ("POST" if body else "GET")
+        request = (f"{method} http://{host}:{port}{path} HTTP/1.1\r\n"
+                   f"Host: {host}:{port}\r\n" + extra)
+        if body:
+            request += f"Content-Length: {len(body)}\r\n"
+        request += "\r\n" + body
+        text, _ = self.raw_session(pport, request, host=phost, quiet=2)
+        return text
+
+    def raw_server(self, port, reply, close_after=True, host="127.0.0.1",
+                   drain=False):
+        """Answer every connection with fixed bytes. Returns a stop function.
+
+        For the shapes a real server would have to be talked into: an answer
+        whose body is delimited by the close, or one which promises to stay
+        and does not. drain reads the whole request first, however large,
+        which is what a test of the sending side needs.
+        """
+        sock = socket.socket()
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        sock.bind((host, port))
+        sock.listen(8)
+        running = [True]
+
+        def serve():
+            while running[0]:
+                try:
+                    conn, _ = sock.accept()
+                except OSError:
+                    break
+                try:
+                    conn.settimeout(0.5 if drain else self.timeout)
+                    while True:
+                        try:
+                            if not conn.recv(65536) or not drain:
+                                break
+                        except socket.timeout:
+                            break       # it has stopped sending
+                    conn.sendall(reply)
+                    if close_after:
+                        conn.close()
+                    else:
+                        self._raw_kept.append(conn)
+                except OSError:
+                    pass
+
+        thread = threading.Thread(target=serve, daemon=True)
+        thread.start()
+
+        def stop():
+            running[0] = False
+            try:
+                sock.close()
+            except OSError:
+                pass
+
+        return stop
+
     # ---- UDP ---------------------------------------------------------
 
     def udp_echo(self, prefix=b"echo:"):