Reverse proxy: sozu

[eshattow]

Reverse proxy: sozu

Hello, I share some experience about Stalwart with sozu reverse proxy on a Debian 12 virtual private server that has limited memory resources.

Git repository for sozu contains some documentation

https://github.com/sozu-proxy/sozu

Install

n.b. some Rust language tools are present in Debian 13 (codename: "trixie") and missing in Debian 12 (codename: "bookworm"). For this situation of Debian 12 on VPS provider and no Debian 13 base installation possible, the package repository of Debian 13 is added and pinned at a minimal priority:

cat > /etc/apt/preferences.d/debian-trixie.pref << EOF
Package: *
Pin: release o=Debian,n=trixie
Pin-Priority: -1

Package: *
Pin: release o=Debian,n=trixie-security
Pin-Priority: -1

Package: *
Pin: release o=Debian,n=trixie-updates
Pin-Priority: -2

Package: *
Pin: release o=Debian,n=trixie-proposed-updates
Pin-Priority: -2
EOF

cat > /etc/apt/sources.list.d/debian-trixie.sources << EOF
Types: deb deb-src
URIs: http://deb.debian.org/debian
Suites: trixie trixie-updates trixie-proposed-updates
Components: main contrib non-free non-free-firmware
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg

Types: deb deb-src
URIs: http://deb.debian.org/debian-security
Suites: trixie-security
Components: main contrib non-free non-free-firmware
Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
EOF

Installing rust stable to build sozu as new user "sozu":

apt install rustup -t trixie
adduser sozu
su sozu
rustup update stable
cat >> $HOME/.bashrc << EOF
# set PATH so it includes user's cargo bin if it exists
if [ -d "$HOME/.cargo/bin" ] ; then
    PATH="$HOME/.cargo/bin:$PATH"
fi
EOF
exit

Build sozu as user "sozu":

su sozu
cargo install sozu --jobs 1 --target-dir cargo-install-sozu-output
exit

Configuration

sozu configuration

Service unit:

cat > /etc/systemd/system/sozu.service << EOF
[Unit]
Description=Sozu - A HTTP reverse proxy, configurable at runtime, fast and safe, built in Rust.
Documentation=https://docs.rs/sozu/
After=network-online.target local-fs.target remote-fs.target time-sync.target
Wants=network-online.target local-fs.target remote-fs.target time-sync.target

StartLimitIntervalSec=10
StartLimitBurst=5

[Service]
Type=simple

RuntimeDirectory=sozu
PIDFile=/run/sozu/sozu.pid

WorkingDirectory=/home/sozu
ExecStartPre=/home/sozu/.cargo/bin/sozu config check --config /home/sozu/config.toml
ExecStart=/home/sozu/.cargo/bin/sozu start --config /home/sozu/config.toml
ExecReload=/home/sozu/.cargo/bin/sozu reload --config /home/sozu/config.toml
Restart=on-failure
User=sozu
AmbientCapabilities=CAP_NET_BIND_SERVICE

LimitNOFILE=65536

# Since systemd 229, should be in [Unit] but in order to support systemd <229,
# it is also supported to have it here.
StartLimitInterval=10
StartLimitBurst=5

[Install]
WantedBy=multi-user.target
EOF

Sample config (content of /home/sozu/config.toml):

log_level = "info"
log_target = "stdout"
command_buffer_size = 16384
max_command_buffer_size = 163840
handle_process_affinity = false
max_connections = 500
worker_count = 1

[[listeners]]
protocol = "http"
address = "[::]:80"

[[listeners]]
protocol = "https"
address = "[::]:443"
certificate = "fullchain.pem"
key = "privkey.pem"
certificate_chain = "fullchain.pem"

[[listeners]]
protocol = "tcp"
address = "[::]:143"
[[listeners]]
protocol = "tcp"
address = "0.0.0.0:143"

[[listeners]]
protocol = "tcp"
address = "[::]:993"
[[listeners]]
protocol = "tcp"
address = "0.0.0.0:993"

[[listeners]]
protocol = "tcp"
address = "[::]:110"
[[listeners]]
protocol = "tcp"
address = "0.0.0.0:110"

[[listeners]]
protocol = "tcp"
address = "[::]:995"
[[listeners]]
protocol = "tcp"
address = "0.0.0.0:995"

[[listeners]]
protocol = "tcp"
address = "[::]:25"
[[listeners]]
protocol = "tcp"
address = "0.0.0.0:25"

[[listeners]]
protocol = "tcp"
address = "[::]:587"
[[listeners]]
protocol = "tcp"
address = "0.0.0.0:587"

[[listeners]]
protocol = "tcp"
address = "[::]:465"
[[listeners]]
protocol = "tcp"
address = "0.0.0.0:465"

[clusters]
[clusters.stalwart]
protocol = "http"
frontends = [
    { address = "[::]:80", hostname = "mail.domain.tld", path = '/', path_type = "PREFIX" },
    { address = "[::]:443", hostname = "mail.domain.tld", certificate = "fullchain.pem", key = "privkey.pem", certificate_chain = "fullchain.pem", },
]
backends = [
    { address = "[::1]:10080" },
]

[clusters.apache2]
protocol = "http"
frontends = [
    { address = "[::]:80", hostname = "domain.tld" },
    { address = "[::]:443", hostname = "domain.tld", certificate = "fullchain.pem", key = "privkey.pem", certificate_chain = "fullchain.pem", },
# Is this a coincidence that REGEX apparently takes precedence over PREFIX?
# see: https://github.com/sozu-proxy/sozu/issues/1222
# Goal here is to re-use the acme-challenge from domain.tld for mail.domain.tld
# and paired with Apache `ServerAlias *.domain.tld` directive so that
# `certbot certonly --webroot --webroot-path /var/www/html` is valid for both
# in one certbot update action to locate /var/www/html/.well-known/acme-challenge/*
    { address = "[::]:80", hostname = "mail.domain.tld", path = '^(\/\.well-known\/acme-challenge)', path_type = "REGEX" },
]
backends = [
    { address = "[::1]:8080" },
]

[clusters.imap]
protocol = "tcp"
send_proxy = true
frontends = [
    { address = "[::]:143" },
    { address = "0.0.0.0:143" },
]
backends = [
    { address = "[::1]:10143" },
]

[clusters.imap]
protocol = "tcp"
send_proxy = true
frontends = [
    { address = "[::]:143" },
    { address = "0.0.0.0:143" },
]
backends = [
    { address = "[::1]:10143" },
]

[clusters.imaps]
protocol = "tcp"
send_proxy = true
frontends = [
    { address = "[::]:993" },
    { address = "0.0.0.0:993" },
]
backends = [
    { address = "[::1]:10993" },
]

[clusters.pop3]
protocol = "tcp"
send_proxy = true
frontends = [
    { address = "[::]:110" },
    { address = "0.0.0.0:110" },
]
backends = [
    { address = "[::1]:10110" },
]

[clusters.pop3s]
protocol = "tcp"
send_proxy = true
frontends = [
    { address = "[::]:995" },
    { address = "0.0.0.0:995" },
]
backends = [
    { address = "[::1]:10995" },
]

[clusters.smtp]
protocol = "tcp"
send_proxy = true
frontends = [
    { address = "[::]:25" },
    { address = "0.0.0.0:25" },
]
backends = [
    { address = "[::1]:10025" },
]

[clusters.submission]
protocol = "tcp"
send_proxy = true
frontends = [
    { address = "[::]:587" },
    { address = "0.0.0.0:587" },
]
backends = [
    { address = "[::1]:10587" },
]

[clusters.submissions]
protocol = "tcp"
send_proxy = true
frontends = [
    { address = "[::]:465" },
    { address = "0.0.0.0:465" },
]
backends = [
    { address = "[::1]:10465" },
]

Script to run certbot:

#!/bin/bash
if [ "$EUID" -ne 0 ]; then
  echo "Please run as root"
  exit
fi
domains=$(grep -oP '(hostname = ")\K[^"]+(?=")' /home/sozu/config.toml | sort | uniq | paste -sd ',')
certbot certonly --keep-until-expiring -d $domains --cert-name domain.tld --webroot --webroot-path /var/www/html
if [ $? -eq 0 ]; then
  set -x
  install -o sozu -g sozu -m 600 -p -t /home/sozu /etc/letsencrypt/live/domain.tld/{fullchain,privkey}.pem
  systemctl restart sozu
  set +x
fi

Note:

• use of https listener protocol configuration with a frontend having http protocol in a cluster, an SSL certificate must be specified by any of the listener protocol configuration or the frontend having http protocol
• clusters that have Proxy Protocol sending enabled with "send_proxy = true" are only applicable to tcp listeners, compared to http (or https) listeners that use forwarding HTTP headers and so are not enabled with Proxy Protocol.
• undefined behavior how to inverted match on http resource paths or what the precedence is for matches

stalwart configuration

assumed "/opt/stalwart" installation path of stalwart editing "/opt/stalwart/etc/config.toml":

...
server.hostname = "host.domain.tld"
server.listener.http.bind = ["[::1]:10080"]
server.listener.http.protocol = "http"
server.listener.https.bind = ["[::1]:10443"]
server.listener.https.protocol = "http"
server.listener.https.tls.implicit = true
server.listener.imap.bind = "[::1]:10143"
server.listener.imap.protocol = "imap"
server.listener.imap.proxy.trusted-networks = ["::1"]
server.listener.imaptls.bind = "[::1]:10993"
server.listener.imaptls.protocol = "imap"
server.listener.imaptls.proxy.trusted-networks = ["::1"]
server.listener.imaptls.tls.implicit = true
server.listener.pop3.bind = "[::1]:10110"
server.listener.pop3.protocol = "pop3"
server.listener.pop3.proxy.trusted-networks = ["::1"]
server.listener.pop3s.bind = "[::1]:10995"
server.listener.pop3s.protocol = "pop3"
server.listener.pop3s.proxy.trusted-networks = ["::1"]
server.listener.pop3s.tls.implicit = true
server.listener.sieve.bind = "[::1]:14190"
server.listener.sieve.protocol = "managesieve"
server.listener.sieve.proxy.trusted-networks = ["::1"]
server.listener.smtp.bind = "[::1]:10025"
server.listener.smtp.protocol = "smtp"
server.listener.smtp.proxy.trusted-networks = ["::1"]
server.listener.submission.bind = "[::1]:10587"
server.listener.submission.protocol = "smtp"
server.listener.submission.proxy.trusted-networks = ["::1"]
server.listener.submissions.bind = "[::1]:10465"
server.listener.submissions.protocol = "smtp"
server.listener.submissions.proxy.trusted-networks = ["::1"]
server.listener.submissions.tls.implicit = true
...

...and so on. Note that Proxy Protocol V2 is active only when trusted-networks is configured; this may be configured per listener as above or more generally. The use of Proxy Protocol V2 in stalwart corresponds with the requirement in the reverse proxy ("tcp" cluster protocol in sozu).

Summary

Reverse proxy sozu is interesting although there are limitations which make the other reverse proxy projects supported in Stalwart to be a better choice for complex installations. The sample configuration above does not result in a functional installation for the objective of stalwart and generic web server on the same host. Hopefully this is interesting start to discussion about including sozu configuration in the Stalwart documentation.