http server implemented, pcre support for hostnames in acls implemented

This commit is contained in:
Vladimir Dubrovin committed 2026-08-29 19:09:54 +03:00
1 parent ea4b2cc3a2
commit 9529a1dfcf
27 files changed
+2325 -148

No files matched your search

+2 -2
View File
@@ -21,7 +21,7 @@ def run(t):
ssl_serv
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{secure}
ssl_noserv"""
@@ -30,7 +30,7 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{plain}
{tls_origin}
+2 -2
View File
@@ -6,14 +6,14 @@ def run(t):
openport = t.free_port()
t.start("httpsrv_auth", f"""
log
http * /echo echo
http echo * /echo
auth strong
users alice:CL:secret bob:CL:hunter2
allow alice
httpsrv -p{srv}
flush
http * /echo echo
http echo * /echo
auth iponly
allow *
httpsrv -p{openport}
+195
View File
@@ -0,0 +1,195 @@
"""The operations that serve a filesystem: file, cache, redir and rewrite.
A rule maps a request onto a path with a template, where $1 upwards stand for
what the stars or the groups of a regular expression matched.
"""
import os
def run(t):
root = os.path.join(t.tmpdir, "web")
os.makedirs(os.path.join(root, "picts", "set"), exist_ok=True)
with open(os.path.join(root, "a.html"), "w") as fp:
fp.write("<h1>hello</h1>")
with open(os.path.join(root, "big.bin"), "wb") as fp:
fp.write(b"x" * 300000) # past a single send, and past the cache limit
with open(os.path.join(root, "picts", "set", "dog.gif"), "wb") as fp:
fp.write(b"GIF89a-pretend")
with open(os.path.join(root, "b.webp"), "wb") as fp:
fp.write(b"RIFF-pretend")
port = t.free_port()
t.start("httpsrv_files", f"""
log
auth iponly
allow *
http rewrite * /alias/** "/w/$1"
http file * /w/*.html "{root}/$1.html"
http file * /big {root}/big.bin
http cache * /c/*.html "{root}/$1.html"
http cache * "pcre:^/(.*)/pic/(.*)\\.(gif|jpeg)$" "{root}/picts/$1/$2.$3"
http redir * /old/** 301 "https://example.org/$1"
http redir * /moved /w/a.html
http file * /rel/*.html "web/$1.html"
http echo * /echo
http_content_type .webp image/webp
http_content_type dat application/x-mydata
http file * /ct/*.webp "{root}/$1.webp"
http file * /named/*.html "{root}/$1.html" text/x-named
http file * /star/*.html "{root}/$1.html" *
http cache * /ctc/*.webp "{root}/$1.webp"
http file * /aged/*.html "{root}/$1.html" * 3600
http cache * /aged2/*.html "{root}/$1.html" * 60
http file * /extra/*.html "{root}/$1.html" * * "X-One: 1\\nX-Two: two words"
http file * /err/*.html "{root}/$1.html" * * "X-Served: static" 404
http reply * /ok**
http reply * /nobody** 204
http reply * /down** 503 "Retry-After: 30"
http cache * /held/*.html "{root}/$1.html" * 30
httpsrv -p{port}
""", ports=[port])
url = f"http://127.0.0.1:{port}"
# --- file -------------------------------------------------------------
r = t.http(url + "/w/a.html")
t.eq(200, r.status, "a file is served")
t.contains(r, "<h1>hello</h1>", "with its content")
t.eq("text/html", r.header("Content-Type"), "and a type taken from the name")
t.eq(str(len("<h1>hello</h1>")), r.header("Content-Length"), "and its length")
t.eq(404, t.http(url + "/w/nosuch.html").status, "a missing file is not found")
big = t.http(url + "/big")
t.eq(300000, big.length, "a large file arrives whole")
t.eq("300000", big.header("Content-Length"), "and is announced by its length")
t.eq(None, big.header("Transfer-Encoding"),
"a file is never sent chunked")
t.eq("300000", t.http(url + "/big", method="HEAD").header("Content-Length"),
"HEAD gives the length without the body")
t.eq(200, t.http(url + "/w/a.html", method="HEAD").status, "HEAD is answered")
t.eq(0, t.http(url + "/w/a.html", method="HEAD").length, "HEAD carries no body")
# --- cache ------------------------------------------------------------
first = t.http(url + "/c/a.html")
second = t.http(url + "/c/a.html")
t.eq(200, first.status, "a cached file is served")
t.eq(first.text, second.text, "and the same on the next request")
t.contains(second, "<h1>hello</h1>", "from memory this time")
# a file changed on disk is noticed rather than served from before
with open(os.path.join(root, "a.html"), "w") as fp:
fp.write("<h1>changed</h1>")
t.contains(t.http(url + "/c/a.html"), "changed",
"a file replaced on disk is read again")
# --- what the stars stand for -----------------------------------------
r = t.http(url + "/set/pic/dog.gif")
t.eq(200, r.status, "a regular expression maps a request onto a path")
t.contains(r, "GIF89a", "and the file is served")
t.eq("image/gif", r.header("Content-Type"), "with the type of that name")
# --- redir ------------------------------------------------------------
r = t.http(url + "/old/thing")
t.eq(301, r.status, "a redirect uses the status it was given")
t.eq("https://example.org/thing", r.header("Location"),
"and a location built from the request")
t.eq(302, t.http(url + "/moved").status, "without a status it is 302")
# --- rewrite ----------------------------------------------------------
r = t.http(url + "/alias/a.html")
t.eq(200, r.status, "a rewritten request reaches the rule after it")
t.contains(r, "changed", "and is served from the path it was rewritten to")
# --- the type a reply carries -------------------------------------------
# Worked out from the name, using what the configuration has registered
# on top of what is built in, unless the rule says otherwise.
t.eq("image/webp", t.http(url + "/ct/b.webp").header("Content-Type"),
"a registered extension names the type")
t.eq("image/webp", t.http(url + "/ctc/b.webp").header("Content-Type"),
"and a cached file is answered the same way")
t.eq("text/x-named", t.http(url + "/named/a.html").header("Content-Type"),
"a rule may name the type itself")
t.eq("text/html", t.http(url + "/star/a.html").header("Content-Type"),
"and a star there leaves it to the name of the file")
# --- what a rule adds to the answer ------------------------------------
r = t.http(url + "/aged/a.html")
t.eq("max-age=3600", r.header("Cache-Control"), "a rule may describe caching")
t.eq("max-age=60", t.http(url + "/aged2/a.html").header("Cache-Control"),
"a cached file is answered the same way")
t.eq(None, t.http(url + "/w/a.html").header("Cache-Control"),
"and a rule which says nothing sends nothing")
r = t.http(url + "/extra/a.html")
t.eq("1", r.header("X-One"), "a rule may add headers")
t.eq("two words", r.header("X-Two"),
"the second of them arrives whole, spaces and all")
r = t.http(url + "/err/a.html")
t.eq(404, r.status, "a rule may answer with the status it names")
t.contains(r, "<h1>", "and the file is still the body")
t.eq("static", r.header("X-Served"),
"what the rule adds goes with the status the rule asked for")
# a refusal the server decided on is its own answer
r = t.http(url + "/err/nosuch.html")
t.eq(404, r.status, "a missing file is still not found")
t.eq(None, r.header("X-Served"), "and carries none of the rule's headers")
t.eq(None, t.http(url + "/aged/nosuch.html").header("Cache-Control"),
"nor what it said about caching")
# --- reply --------------------------------------------------------------
r = t.http(url + "/ok")
t.eq(200, r.status, "reply answers with 200 by default")
t.eq("0", r.header("Content-Length"), "with a length of zero")
t.eq(0, r.length, "and no body")
r = t.http(url + "/nobody")
t.eq(204, r.status, "reply answers with the status it was given")
t.eq(None, r.header("Content-Length"),
"and a status carrying no body is sent without a length")
r = t.http(url + "/down")
t.eq(503, r.status, "reply serves a refusal the configuration decided on")
t.eq("30", r.header("Retry-After"), "with the headers that go with it")
# --- a client which has the file already --------------------------------
r = t.http(url + "/w/a.html")
stamp = r.header("Last-Modified")
t.ne(None, stamp, "a file is answered with the time it was last changed")
r = t.http(url + "/w/a.html", headers={"If-Modified-Since": stamp})
t.eq(304, r.status, "and an unchanged file is answered 304")
t.eq(0, r.length, "which carries no body")
t.eq(None, r.header("Content-Length"), "and no length")
t.eq(stamp, r.header("Last-Modified"), "but still says when the file changed")
t.eq(200, t.http(url + "/w/a.html",
headers={"If-Modified-Since": "Sun, 06 Nov 1994 08:49:37 GMT"}).status,
"an older date is answered with the file")
t.eq(200, t.http(url + "/w/a.html",
headers={"If-Modified-Since": "not a date at all"}).status,
"and a date which cannot be read is treated as none")
t.eq(304, t.http(url + "/c/a.html", headers={"If-Modified-Since": stamp}).status,
"a file answered from memory is conditional in the same way")
t.eq(404, t.http(url + "/err/a.html", headers={"If-Modified-Since": stamp}).status,
"a rule with a status of its own is not turned into a 304")
# --- a rule which says how long its copy may be held --------------------
t.contains(t.http(url + "/held/a.html"), "changed", "a held file is served")
with open(os.path.join(root, "a.html"), "w") as fp:
fp.write("<h1>replaced</h1>")
t.not_contains(t.http(url + "/held/a.html"), "replaced",
"and within its max-age the disk is not looked at again")
t.contains(t.http(url + "/c/a.html"), "replaced",
"while a rule without one notices the change at once")
# --- the paths a rule may not build ------------------------------------
t.eq(403, t.http(url + "/rel/a.html").status,
"a relative target is refused")
t.ne(200, t.http(url + "/w/../etc/passwd").status,
"a request climbing out of the tree is refused")
+109
View File
@@ -0,0 +1,109 @@
"""Keep-alive: which answers may be followed by another request.
The next request begins where the last answer ended, so a connection is only
kept when the length of what was sent is known exactly and the body of the
request was read to its end. Everything else closes, which is the safe way to
be wrong.
"""
import os
def run(t):
root = os.path.join(t.tmpdir, "ka")
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "a.html"), "w") as fp:
fp.write("<h1>hello</h1>")
port = t.free_port()
t.start("httpsrv_keepalive", f"""
log
auth iponly
allow *
http file * /w/*.html "{root}/$1.html"
http reply * /ok** 200
http echo * /echo**
http data * /chunked** size=100&chunked=1
httpsrv -p{port}
""", ports=[port])
def req(path, version="1.1", extra="", body=""):
head = (f"GET {path} HTTP/{version}\r\nHost: t\r\n{extra}\r\n")
if body:
head = head.replace("GET", "POST", 1)
return head + body
def session(*requests, quiet=0.5):
text, closed = t.raw_session(port, "".join(requests), quiet=quiet)
return text, closed, text.count("HTTP/1.")
# --- what keeps the connection ---------------------------------------
text, closed, n = session(req("/w/a.html"), req("/ok"),
req("/w/a.html", extra="Connection: close\r\n"))
t.eq(3, n, "three 1.1 requests are answered on one connection")
t.eq(True, closed, "and the one asking to close ends it")
t.contains(text, "Connection: keep-alive", "the answers say the connection is kept")
t.eq(2, text.count("<h1>hello</h1>"), "each file arrives whole")
text, closed, n = session(req("/w/a.html", version="1.0"), req("/ok", version="1.0"))
t.eq(1, n, "a 1.0 request without the header is answered once")
t.eq(True, closed, "and the connection ends")
t.contains(text, "Connection: close", "which the answer says")
text, closed, n = session(req("/w/a.html", version="1.0",
extra="Connection: keep-alive\r\n"),
req("/ok", version="1.0",
extra="Connection: close\r\n"))
t.eq(2, n, "a 1.0 client asking for keep-alive gets it")
# a request carrying a body: the next one begins after it
text, closed, n = session(req("/echo", extra="Content-Length: 5\r\n", body="hello"),
req("/ok", extra="Connection: close\r\n"))
t.eq(2, n, "a body which was read to its end leaves the stream in place")
t.contains(text, "content.length=5", "and the body was seen")
# --- what ends it -----------------------------------------------------
text, closed, n = session(req("/echo", extra="Transfer-Encoding: chunked\r\n"),
req("/ok"))
t.eq(1, n, "a request body this server cannot frame ends the connection")
t.eq(True, closed, "the connection is closed rather than left mid-body")
# A body longer than the server is willing to read leaves the rest of it
# in the stream, so the connection cannot carry another request. The send
# may not even finish - the server answers and closes part way through -
# which is the same answer from the other side.
big = "x" * 1500000
text, closed, n = session(req("/echo", extra="Content-Length: 1500000\r\n", body=big),
quiet=2)
t.eq(1, n, "a body past what the server will read is answered once")
t.contains(text, "Connection: close",
"and the answer ends the connection rather than leaving the rest to be read")
# --- answers of other shapes -----------------------------------------
text, closed, n = session(req("/chunked"), req("/ok", extra="Connection: close\r\n"))
t.eq(2, n, "a chunked answer may be followed by another request")
text, closed, n = session(req("/chunked", version="1.0"), req("/ok", version="1.0"))
t.eq(1, n, "but not for a client which has no chunked encoding to read")
stamp = t.http(f"http://127.0.0.1:{port}/w/a.html").header("Last-Modified")
text, closed, n = session(req("/w/a.html", extra=f"If-Modified-Since: {stamp}\r\n"),
req("/ok", extra="Connection: close\r\n"))
t.eq(2, n, "a 304 carries no body and the next request follows it")
t.contains(text, "304", "and it is a 304")
# --- the administration pages always close ---------------------------
aport = t.free_port()
t.start("httpsrv_keepalive_admin", f"""
log
auth iponly
allow *
http file * /w/*.html "{root}/$1.html"
admin -p{aport}
""", ports=[aport])
text, closed = t.raw_session(aport,
f"GET /w/a.html HTTP/1.1\r\nHost: t\r\n\r\nGET /C HTTP/1.1\r\nHost: t\r\n\r\n"
f"GET /w/a.html HTTP/1.1\r\nHost: t\r\n\r\n")
t.eq(2, text.count("HTTP/1."), "an administration page is the last thing on a connection")
t.eq(True, closed, "which the server closes, since the page states no length")
+3 -3
View File
@@ -9,9 +9,9 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http * /data data
http * /small data size=64
http echo * /echo**
http data * /data
http data * /small size=64
httpsrv -p{srv}
""", ports=[srv])
+27 -2
View File
@@ -11,8 +11,8 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http * /safe/* echo
http echo * /echo**
http echo * /safe/**
httpsrv -p{srv}
""", ports=[srv])
@@ -62,3 +62,28 @@ def run(t):
headers={"Content-Type": "application/x-www-form-urlencoded"})
t.contains(r, "method=POST", "POST reaches the handler")
t.contains(r, "content.length=9", "the POST content length is parsed")
# --- dollars in the configuration ------------------------------------
# Outside quotes a dollar begins the name of a file to include, so an
# argument holding one is quoted. Two dollars stand for one, which is how
# a dollar reaches a rule as text.
dsrv = t.free_port()
t.start("httpsrv_dollar", f"""
log
auth iponly
allow *
http redir * /old** 301 "http://example.org/x$$y/$1"
http redir * "pcre:^/re/([a-z]+)$" 302 "http://example.org/re/$1"
http echo * /**
httpsrv -p{dsrv}
""", ports=[dsrv])
durl = f"http://127.0.0.1:{dsrv}"
r = t.http(durl + "/old/a")
t.eq(301, r.status, "a rule holding a doubled dollar loads")
t.eq("http://example.org/x$y//a", r.header("Location"),
"and two dollars reach the location as one")
t.eq(302, t.http(durl + "/re/abc").status,
"a quoted regular expression keeps its anchor")
t.eq(200, t.http(durl + "/re/ab9").status,
"and the anchor is real: what it excludes falls through")
+39 -10
View File
@@ -8,19 +8,25 @@ def run(t):
log
auth iponly
allow *
http * /exact echo
http * /pre* echo
http * *.suffix echo
http * *mid* echo
http host.example.com /byhost echo
http *.wild.example.com /bywild echo
http * /only-first echo
http echo * /exact
http echo * /pre*
http echo * /deep/**
http echo * **.suffix
http echo * **mid**
http echo host.example.com /byhost
http echo *.wild.example.com /bywild
http echo * /only-first
http rewrite_host *.old.example ** "$1.new.example"
http rewrite_host "pcre:^legacy-(.*)$" ** "$1.new.example"
http rewrite_host * /badhost** "not a host name"
http echo one.new.example /**
httpsrv -p{srv}
flush
auth iponly
allow *
http * /only-second echo
http echo * /only-second
httpsrv -p{srv2}
""", ports=[srv, srv2])
@@ -31,8 +37,15 @@ def run(t):
t.eq(404, t.http(url + "/exactly").status,
"an exact URL does not match a longer path")
t.eq(200, t.http(url + "/pre").status, "a prefix matches the bare prefix")
t.eq(200, t.http(url + "/pretty/deep").status,
"a prefix matches a longer path")
t.eq(200, t.http(url + "/pretty").status,
"a prefix matches a longer name in the same path element")
# a single star stays inside one element of the path, which is what keeps
# a rule from reaching into directories it did not name
t.eq(404, t.http(url + "/pretty/deep").status,
"a prefix does not cross a slash")
t.eq(200, t.http(url + "/deep/a/b/c").status,
"a double star does cross one")
t.eq(200, t.http(url + "/any.suffix").status, "a suffix matches")
t.eq(404, t.http(url + "/any.suffixx").status,
"a suffix is anchored at the end")
@@ -67,3 +80,19 @@ def run(t):
"the second service has its own rules")
t.eq(404, t.http(f"http://127.0.0.1:{srv2}/only-first").status,
"the second service does not have the earlier rules")
# --- a rule which changes the host --------------------------------
# The stars of the host pattern are what $1 upwards stand for here, the
# way the stars of the URL stand for themselves in a rewrite.
r = t.http(url + "/anything", headers={"Host": "one.old.example"})
t.eq(200, r.status, "a rewritten host reaches the rules after it")
t.contains(r, "host=one.new.example", "and the request carries the new name")
t.eq(200, t.http(url + "/anything", headers={"Host": "legacy-one"}).status,
"a regular expression names the part to keep")
t.eq(404, t.http(url + "/anything", headers={"Host": "other.example"}).status,
"a host no rule rewrites is left as it was")
t.eq(403, t.http(url + "/badhost", headers={"Host": "x"}).status,
"a rule may not build something which is not a host name")
+3 -3
View File
@@ -21,8 +21,8 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http * /data data
http echo * /echo**
http data * /data
httpsrv -p{origin} -i::1
# reached over IPv6, and allowed to reach IPv6
@@ -102,7 +102,7 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{v4origin}
{"".join(sections)}
""", ports=[v4origin] + list(family_ports.values()))
+1 -1
View File
@@ -43,7 +43,7 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{srv}
# every outgoing connection binds inside the range
+25 -4
View File
@@ -29,9 +29,9 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http * /secret* echo
http * /data data
http echo * /echo**
http echo * /secret**
http data * /data
httpsrv -p{origin}
""", ports=[origin])
@@ -88,6 +88,27 @@ def run(t):
t.eq(200, t.http(url + "/echo", proxy=p).status,
"an extension that matches nothing changes nothing")
# --- a regular expression where a host name is expected -----------------
# The same prefix works in an access rule and in an http rule, so one
# kind of expression is understood wherever a name can be written.
named = t.free_port()
t.start("pcre_named", f"""
log
flush
nserver 127.0.0.1
nscache 1024
nsrecord host1.test 127.0.0.1
nsrecord other.test 127.0.0.1
auth iponly
allow * * "pcre:^host[0-9]+\\.test$"
proxy -p{named}
""", ports=[named])
t.eq(200, t.http(f"http://host1.test:{origin}/echo", proxy=f"127.0.0.1:{named}").status,
"a destination matching the expression is allowed")
t.ne(200, t.http(f"http://other.test:{origin}/echo", proxy=f"127.0.0.1:{named}").status,
"one that does not match is refused")
# --- rewriting the reply ------------------------------------------------
p = proxy_with("rewrite_srv",
'pcre_rewrite srvheader dunno "text/plain" "text/rewritten"',
@@ -143,7 +164,7 @@ def run(t):
flush
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{elsewhere}
""", ports=[elsewhere])
+2 -2
View File
@@ -11,8 +11,8 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http * /data data
http echo * /echo**
http data * /data
httpsrv -p{origin}
flush
+3 -3
View File
@@ -16,15 +16,15 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http * /data data
http echo * /echo**
http data * /data
httpsrv -p{srv}
# a second origin, used as a destination the rules must keep out
flush
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{other}
# an open proxy
+2 -2
View File
@@ -10,8 +10,8 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http * /data data
http echo * /echo**
http data * /data
httpsrv -p{srv}
flush
+4 -4
View File
@@ -32,7 +32,7 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{origin}
flush
@@ -76,8 +76,8 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http * /data data
http echo * /echo**
http data * /data
httpsrv -p{origin}
flush
@@ -129,7 +129,7 @@ def run(t):
ssl_serv
auth iponly
allow *
http * /secret* echo
http echo * /secret**
httpsrv -p{origin}
""", ports=[origin])
+1 -1
View File
@@ -17,7 +17,7 @@ def run(t):
ssl_serv
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{origin}
flush
+2 -2
View File
@@ -97,14 +97,14 @@ def run(t):
log
auth iponly
allow *
http * /echo* echo
http echo * /echo**
httpsrv -p{origin_port} -i{ORIGIN_ADDR}
# a second server, to tell apart where a connection actually went
flush
auth iponly
allow *
http * * data size=13
http data * * size=13
httpsrv -p{decoy_port} -i{DECOY_ADDR}
# a port mapper aimed at the decoy: with the destination taken from
+36
View File
@@ -321,6 +321,42 @@ class Tester:
except OSError as exc:
return f"<no reply: {exc}>"
def raw_session(self, port, request, host="127.0.0.1", quiet=0.5):
"""Send bytes and read until the server closes or goes quiet.
Returns (text, closed). closed says the server ended the connection
rather than leaving it open for another request, which is the whole
question a keep-alive test asks.
"""
if not isinstance(request, bytes):
request = request.encode("latin-1")
closed = False
chunks = []
try:
with socket.create_connection((host, port), self.timeout) as sock:
try:
sock.sendall(request)
except OSError:
# the server answered and closed before taking all of it,
# which is an answer in itself
closed = True
sock.settimeout(quiet)
while True:
try:
piece = sock.recv(65536)
except socket.timeout:
break # quiet: the connection is still open
except OSError:
closed = True # reset: it is not
break
if not piece:
closed = True
break
chunks.append(piece)
except OSError as exc:
return f"<no reply: {exc}>", True
return b"".join(chunks).decode("utf-8", "replace"), closed
# ---- UDP ---------------------------------------------------------
def udp_echo(self, prefix=b"echo:"):