markuze/haproxy fullnode docker

docker/compose/aptos-node/docker-compose-fullnode.yaml

@@ -3,6 +3,28 @@
 
 version: "3.8"
 services:
+  haproxy:
+    image: haproxytech/haproxy-debian:2.2
+    volumes:
+      - type: bind
+        source: ./haproxy-fullnode.cfg
+        target: /usr/local/etc/haproxy/haproxy.cfg
+      - type: bind
+        source: ./blocked.ips
+        target: /usr/local/etc/haproxy/blocked.ips
+    networks:
+      - shared
+    expose:
+      - 6182
+      - 9101
+      - 9102
+      - 80
+    ports:
+      - "6182:6182"
+      - "80:80"
+      - "9101:9101"
+      - "9102:9102"
+
   fullnode:
     image: "${VALIDATOR_IMAGE_REPO:-aptoslabs/validator}:${IMAGE_TAG:-testnet}"
     networks:
@@ -25,10 +47,6 @@ services:
         target: /opt/aptos/genesis/validator-full-node-identity.yaml
     command: ["/usr/local/bin/aptos-node", "-f", "/opt/aptos/etc/fullnode.yaml"]
     restart: unless-stopped
-    ports:
-      - "6182:6182"
-      - "80:8080"
-      - "9101:9101"
     expose:
       - 6182
       - 80

docker/compose/aptos-node/haproxy-fullnode.cfg

@@ -0,0 +1,102 @@
+global
+    log stdout len 10240 format raw local0
+
+    # Config manual: https://cbonte.github.io/haproxy-dconv/2.5/configuration.html
+    # magic values : terraform/helm/aptos-node/values.yaml
+
+    maxconn 8192
+    # This limits the whole HA Proxy impacting both fullnodes and other frontends
+    # maxconnrate 128
+    nbthread 4
+
+    #4MB for client facing sndbuf/rcvbuf. -- 100Mb/s with 300 mili latency (e.g., us-asia)
+    tune.sndbuf.client 4194304	#tcpBufSize
+    tune.rcvbuf.client 4194304	#tcpBufSize
+
+    user nobody
+
+## TCP port defaults
+defaults
+    log global
+    mode tcp
+    #option tcplog
+    option dontlog-normal
+    log-format "%ci:%cp - %sp[%rt] [%t] %ft %Tw/%Tc/%Tt %B [%ts] %ac/%fc/%bc/%sc/%rc %sq/%bq"
+    maxconn 8192		#Validator network mesh + FN x2
+    retries 3
+    timeout queue 5s  #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/
+    timeout connect 5s
+    # enough for 1 successfull + 5 unsuccessfull HB(10 sec interval) + 20 sec timeout
+    timeout server 80s
+    timeout client 80s
+
+    timeout client-fin 3s #How long to hold an interrupted client connection.
+    timeout server-fin 1s
+
+frontend fullnode-fn
+    mode tcp
+    bind :6182
+    default_backend fullnode
+
+    # Deny requests from blocked IPs
+    tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+
+backend fullnode
+    default-server maxconn 128
+    server fullnode fullnode:6182
+
+
+##################  HTTP: metrics & API
+defaults
+        retries 3
+        timeout queue 5s  #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/
+        timeout connect 5s
+        timeout server 60s #what makes sense? for silence between nodes?
+        timeout client 60s
+
+        timeout client-fin 3s #How long to hold an interrupted client connection.
+        timeout server-fin 1s
+
+	timeout http-request 60s #len of http request
+	timeout http-keep-alive 2s
+
+        rate-limit sessions 256
+
+frontend fullnode-metrics
+    mode http
+    option httplog
+    bind :9101
+    default_backend fullnode-metrics
+
+    # Deny requests from blocked IPs
+    tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+    http-request add-header Forwarded "for=%ci"
+
+backend fullnode-metrics
+    mode http
+    default-server maxconn 16
+    server fullnode fullnode:9101
+
+frontend fullnode-api
+    mode http
+    option httplog
+    bind :8180
+    default_backend fullnode-api
+
+    # Deny requests from blocked IPs
+    tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+    http-request add-header Forwarded "for=%ci"
+
+backend fullnode-api
+    mode http
+    default-server maxconn 16
+    server fullnode fullnode:8080
+
+frontend stats
+    mode http
+    bind :9102
+    option http-use-htx
+    http-request use-service prometheus-exporter if { path /metrics }
+    stats enable
+    stats uri /stats
+    monitor-uri /healthy