3proxy/tests/cases/pcre.py

"""PCRE filtering: matching, rewriting, options and rule scope.

A request rewrite is applied to the buffer the server is sent, so it works
on a direct connection as well as through a parent. The destination was
chosen, and the access rules applied to it, before the filter ran, so a
rewrite that moves the request to another host or changes the method is
ignored rather than acted on.
"""


def _has_pcre(t):
    """Whether this build accepts the pcre commands at all.

    The last line is nonsense on purpose: it makes 3proxy report and exit
    instead of waiting, and what it says about the line above is the answer.
    """
    out = t.run_config("pcre_probe",
                       'log\npcre request deny "x"\nnot_a_command\n')
    return "'pcre'" not in out


def run(t):
    if not _has_pcre(t):
        t.skip("PCRE (this build has no PCRE support)")
        return

    origin = t.free_port()
    t.start("pcre_origin", f"""
        log
        auth iponly
        allow *
        http echo * /echo**
        http echo * /secret**
        http data * /data
        httpsrv -p{origin}
    """, ports=[origin])

    url = f"http://127.0.0.1:{origin}"

    def proxy_with(name, *rules):
        port = t.free_port()
        t.start(name, "\n".join([
            "log", "flush", "auth iponly", "allow *", *rules, f"proxy -p{port}"]),
            ports=[port])
        return f"127.0.0.1:{port}"

    # --- matching and denial ---------------------------------------------
    p = proxy_with("deny", 'pcre request deny "/secret"')
    t.eq(200, t.http(url + "/echo", proxy=p).status, "an unmatched request passes")
    t.ne(200, t.http(url + "/secret/page", proxy=p).status, "a matched request is denied")

    # the rules are ordered, and the first decision wins
    p = proxy_with("allow_first", 'pcre request allow "/echo"', 'pcre request deny "/"')
    t.eq(200, t.http(url + "/echo", proxy=p).status, "allow short-circuits a later deny")
    p = proxy_with("deny_first", 'pcre request deny "/"', 'pcre request allow "/echo"')
    t.ne(200, t.http(url + "/echo", proxy=p).status, "deny short-circuits a later allow")

    # --- what the pattern is matched against ------------------------------
    p = proxy_with("cliheader", 'pcre cliheader deny "BadBot"')
    t.eq(200, t.http(url + "/echo", proxy=p).status, "a header rule ignores other requests")
    t.ne(200, t.http(url + "/echo", proxy=p, headers={"User-Agent": "BadBot/1.0"}).status,
         "a client header can be matched")

    # --- options ------------------------------------------------------------
    p = proxy_with("caseless", "pcre_options PCRE2_CASELESS",
                   'pcre request deny "/SECRET"')
    t.ne(200, t.http(url + "/secret/page", proxy=p).status,
         "PCRE2_CASELESS makes the match case-insensitive")
    p = proxy_with("cased", 'pcre request deny "/SECRET"')
    t.eq(200, t.http(url + "/secret/page", proxy=p).status,
         "without it the match is case-sensitive")

    # --- the access rule a pcre rule carries --------------------------------
    p = proxy_with("ace_here", f'pcre request deny "/echo" * * * {origin}')
    t.ne(200, t.http(url + "/echo", proxy=p).status,
         "a rule applies where its access rule matches")
    p = proxy_with("ace_elsewhere", 'pcre request deny "/echo" * * * 1')
    t.eq(200, t.http(url + "/echo", proxy=p).status,
         "and not where it does not")

    # pcre_extend appends another access rule to the one just defined
    p = proxy_with("extend", 'pcre request deny "/echo" * * * 1',
                   f"pcre_extend * * * {origin}")
    t.ne(200, t.http(url + "/echo", proxy=p).status,
         "pcre_extend widens the rule to another destination")
    p = proxy_with("extend_other", 'pcre request deny "/echo" * * * 1',
                   "pcre_extend * * * 2")
    t.eq(200, t.http(url + "/echo", proxy=p).status,
         "an extension that matches nothing changes nothing")

    # --- a regular expression where a host name is expected -----------------
    # The same prefix works in an access rule and in an http rule, so one
    # kind of expression is understood wherever a name can be written.
    named = t.free_port()
    t.start("pcre_named", f"""
        log
        flush
        nserver 127.0.0.1
        nscache 1024
        nsrecord host1.test 127.0.0.1
        nsrecord other.test 127.0.0.1
        auth iponly
        allow * * "pcre:^host[0-9]+\\.test$"
        proxy -p{named}
    """, ports=[named])

    t.eq(200, t.http(f"http://host1.test:{origin}/echo", proxy=f"127.0.0.1:{named}").status,
         "a destination matching the expression is allowed")
    t.ne(200, t.http(f"http://other.test:{origin}/echo", proxy=f"127.0.0.1:{named}").status,
         "one that does not match is refused")

    # --- rewriting the reply ------------------------------------------------
    p = proxy_with("rewrite_srv",
                   'pcre_rewrite srvheader dunno "text/plain" "text/rewritten"',
                   'pcre_rewrite srvdata dunno "peer.addr" "PEER.ADDR"')
    r = t.http(url + "/echo", proxy=p)
    t.eq(200, r.status, "a rewritten reply still arrives")
    t.eq("text/rewritten", r.header("Content-Type"), "a reply header can be rewritten")
    t.contains(r, "PEER.ADDR", "reply data can be rewritten")
    t.not_contains(r, "peer.addr", "the original text is gone")

    # --- rewriting the request ------------------------------------------------
    p = proxy_with("rewrite_req", 'pcre_rewrite request dunno "/echo/old" "/echo/new"')
    r = t.http(url + "/echo/old", proxy=p)
    t.eq(200, r.status, "a rewritten request still arrives")
    t.contains(r, "path=/echo/new", "the origin sees the rewritten path")

    # the replacement may be longer or shorter than what it replaces
    p = proxy_with("rewrite_long", 'pcre_rewrite request dunno "/echo/x" "/echo/deeper/still"')
    t.contains(t.http(url + "/echo/x", proxy=p), "path=/echo/deeper/still",
               "a longer replacement is spliced in")
    p = proxy_with("rewrite_short", 'pcre_rewrite request dunno "/echo/aaaaaaaaaa" "/echo/b"')
    t.contains(t.http(url + "/echo/aaaaaaaaaa", proxy=p), "path=/echo/b",
               "a shorter replacement is spliced in")

    p = proxy_with("rewrite_query", 'pcre_rewrite request dunno "token=old" "token=new"')
    t.contains(t.http(url + "/echo?token=old", proxy=p), "query=token=new",
               "the query can be rewritten")

    p = proxy_with("rewrite_none", 'pcre_rewrite request dunno "/nothing" "/else"')
    t.contains(t.http(url + "/echo/keep", proxy=p), "path=/echo/keep",
               "a request that does not match is left alone")

    # what follows the request line has to survive the splice
    p = proxy_with("rewrite_post", 'pcre_rewrite request dunno "/echo/old" "/echo/new"')
    r = t.http(url + "/echo/old", proxy=p, method="POST", body="hello",
               headers={"Content-Type": "text/plain"})
    t.contains(r, "path=/echo/new", "a POST is rewritten too")
    t.contains(r, "content.length=5", "its body is still described correctly")

    conn = t.connection("127.0.0.1", origin, proxy=p)
    try:
        first = t.http(url + "/echo/old", proxy=p, conn=conn)
        second = t.http(url + "/echo/old", proxy=p, conn=conn)
        t.contains(first, "path=/echo/new", "the first of two on a connection is rewritten")
        t.contains(second, "path=/echo/new", "and so is the second")
    finally:
        conn.close()

    # --- rewrites that would change where the request goes --------------------
    elsewhere = t.free_port()
    t.start("pcre_elsewhere", f"""
        log
        flush
        auth iponly
        allow *
        http echo * /echo**
        httpsrv -p{elsewhere}
    """, ports=[elsewhere])

    p = proxy_with("rewrite_host",
                   f'pcre_rewrite request dunno "127.0.0.1:{origin}" "127.0.0.1:{elsewhere}"')
    r = t.http(url + "/echo", proxy=p)
    t.eq(200, r.status, "a rewrite naming another host still answers")
    t.contains(r, f"host=127.0.0.1:{origin}",
               "but the request goes where the access rules allowed")

    p = proxy_with("rewrite_method", 'pcre_rewrite request dunno "^GET" "HEAD"')
    t.contains(t.http(url + "/echo", proxy=p), "method=GET",
               "a rewrite of the method is ignored")

    # --- and the same rewrite through an HTTP parent --------------------------
    parent = t.free_port()
    t.start("pcre_parent", f"""
        log
        flush
        auth iponly
        allow *
        proxy -p{parent}
    """, ports=[parent])
    p = proxy_with("rewrite_parent", 'pcre_rewrite request dunno "/echo/old" "/echo/new"',
                   f"parent 1000 http 127.0.0.1 {parent}")
    r = t.http(url + "/echo/old", proxy=p)
    t.eq(200, r.status, "a rewritten request through a parent arrives")
    t.contains(r, "path=/echo/new", "the origin sees the rewritten path through a parent")

    # --- a rewrite which grows the headers ----------------------------------
    # GHSA-h845-prxq-ww3q: a rewrite that doubles the client headers used to
    # leave a buffer holding exactly what it produced, and the Content-Length
    # the data filter regenerates was then written past the end of it.
    # The origin here reads whatever it is sent and answers the same way every
    # time: what is being tested is the proxy in the middle, not what a server
    # is willing to accept in one request.
    grown = t.free_port()
    stop = t.raw_server(grown, b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok",
                        drain=True)
    try:
        p = proxy_with("rewrite_grow",
                       'pcre_rewrite cliheader dunno "(?s).*" "$0$0"',
                       'pcre clidata dunno *')
        big = "".join("X-%d: %s\r\n" % (i, chr(65 + i) * 20000) for i in range(5))
        reply = t.raw_proxy_request(p, f"http://127.0.0.1:{grown}/x",
                                    extra=big, body="z")
        t.contains(reply, "200", "a doubled header block with a body is answered")
        t.contains(t.raw_proxy_request(p, f"http://127.0.0.1:{grown}/x"), "200",
                   "and the proxy is still there afterwards")
    finally:
        stop()

    # A reference to a group the pattern does not have is dropped, and dropped
    # by both the pass which measures the result and the pass which writes it.
    p = proxy_with("rewrite_nogroup",
                   'pcre_rewrite cliheader dunno "(?s)Host:" "$9$9$9$9$9$9$9$9"')
    r = t.http(url + "/echo", proxy=p, headers={"X-Pad": "P" * 2000})
    t.eq(200, r.status, "a reference to a group which did not match is left out")
    t.contains(t.http(url + "/echo", proxy=p), "path=/echo",
               "and that proxy is still there too")

    # an optional group which took part on one request and not on the next
    p = proxy_with("rewrite_optgroup",
                   'pcre_rewrite cliheader dunno "X-Mark: (a)?(b)" "[$1][$2]"')
    t.eq(200, t.http(url + "/echo", proxy=p, headers={"X-Mark": "ab"}).status,
         "a group which matched is put in")
    t.eq(200, t.http(url + "/echo", proxy=p, headers={"X-Mark": "b"}).status,
         "and one which did not is left out")