Increasing haproxy resources

docker/compose/aptos-node/blocked.ips

@@ -0,0 +1 @@
+# Add Blocked Ips here

docker/compose/aptos-node/docker-compose.yaml

@@ -3,6 +3,29 @@
 
 version: "3.8"
 services:
+  haproxy:
+    image: haproxytech/haproxy-debian:2.2
+    volumes:
+      - type: bind
+        source: ./haproxy.cfg
+        target: /usr/local/etc/haproxy/haproxy.cfg
+      - type: bind
+        source: ./blocked.ips
+        target: /usr/local/etc/haproxy/blocked.ips
+    networks:
+      - shared
+    expose:
+      - 6180
+      - 6181
+      - 9101
+      - 80
+    ports:
+      - "6180:6180"
+      - "6181:6181"
+      - "80:80"
+      - "9101:9101"
+
+
   validator:
     image: "${VALIDATOR_IMAGE_REPO:-aptoslabs/validator}:${IMAGE_TAG:-testnet}"
     networks:
@@ -25,11 +48,6 @@ services:
         target: /opt/aptos/genesis/validator-identity.yaml
     command: ["/usr/local/bin/aptos-node", "-f", "/opt/aptos/etc/validator.yaml"]
     restart: unless-stopped
-    ports:
-      - "6180:6180"
-      - "6181:6181"
-      - "80:8080"
-      - "9101:9101"
     expose:
       - 6180
       - 6181

docker/compose/aptos-node/haproxy.cfg

@@ -0,0 +1,180 @@
+global
+    log stdout len 10240 format raw local0
+
+    # Config manual: https://cbonte.github.io/haproxy-dconv/2.5/configuration.html
+    # magic values : terraform/helm/aptos-node/values.yaml
+
+    maxconn 1024
+    # This limits the whole HA Proxy impacting both validators and other frontends
+    # maxconnrate 128
+    nbthread 4
+
+    #4MB for client facing sndbuf/rcvbuf. -- 100Mb/s with 300 mili latency (e.g., us-asia)
+    tune.sndbuf.client 4194304	#tcpBufSize
+    tune.rcvbuf.client 4194304	#tcpBufSize
+
+    user nobody
+
+## TCP port defaults
+defaults
+    log global
+    mode tcp
+    #option tcplog
+    option dontlog-normal
+    log-format "%ci:%cp - %sp[%rt] [%t] %ft %Tw/%Tc/%Tt %B [%ts] %ac/%fc/%bc/%sc/%rc %sq/%bq"
+    maxconn 1024		#Validator network mesh + FN x2
+    retries 3
+    timeout queue 5s  #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/
+    timeout connect 5s
+    # enough for 1 successfull + 5 unsuccessfull HB(10 sec interval) + 20 sec timeout
+    timeout server 80s
+    timeout client 80s
+
+    timeout client-fin 3s #How long to hold an interrupted client connection.
+    timeout server-fin 1s
+
+frontend fe-validator
+    bind :6180
+    default_backend validator
+
+    # Deny requests from blocked IPs
+    tcp-request connection silent-drop if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+
+    # We deem a connection rate high when an IP is attempting to reconnect more than twice a min
+    acl ip_high_conn_rate sc0_conn_rate gt 2
+
+    stick-table type ip size 10m expire 30m store gpc0,gpc1,conn_rate(1m),bytes_out_rate(10s),bytes_out_cnt	##about 500MB of memory
+    tcp-request connection track-sc0 src 						   #update table with src ip as key, store in sc0
+
+    #We Count rate-limit manualy -- Will be more CPU intensieve but will allow whitelists to enter and up to rateLimitSession non blacklisted IPs.
+    tcp-request connection track-sc1 int(1) table CONN_RATE
+
+    #tcp-request connection sc-set-gpt0(0) int(...) if ip_high_conn_rate is better but dies with:
+    #parsing [/usr/local/etc/haproxy/haproxy.cfg:53] : internal error, unexpected rule->from=0, please report this bug!
+    #<1> Mark Blacklist
+    tcp-request connection sc-inc-gpc0(0) if ip_high_conn_rate
+
+    #This connection is silently dropped no reason to count it for rateLimitSession
+    tcp-request connection sc-inc-gpc1(1) unless { sc0_get_gpc0() ge 1 }
+
+    # an IP that was blacklisted due to to many unsucsessfull tcp attempts
+    #-1- Enforece Blacklist
+    tcp-request connection silent-drop if { sc0_get_gpc0() ge 1 }
+
+    #an IP that had a sucessfull connection.
+    #-2- Allow Whitelist
+    tcp-request connection accept if { sc0_get_gpc1() ge 1 }
+
+    #-3- Enforece RateLimit. Connection attempts by *new* IPs/sec
+    tcp-request connection reject if { sc1_gpc1_rate(CONN_RATE) gt  256 } #rateLimitSession
+
+    # This is a successfull connection i.e., was sent more than 16K bytes in the last 30 min
+    #tcp-request session sc-set-gpt0(0) int(...)  if { sc0_kbytes_out gt 16 }
+    #<2> Mark Whitelist
+    tcp-request session sc-inc-gpc1(0) if { sc0_kbytes_out gt 4 }
+
+    # -4- Break a long high rate connection
+    # maxBytesOutRate10sec: 100mb/s for 10 sec
+    tcp-request session reject if { sc0_bytes_out_rate gt 134217728 }
+
+backend validator
+    default-server maxconn 1024
+    server validator validator:6180
+
+frontend fe-fullnode
+    bind :6181
+    default_backend validator-fn
+
+    # Deny requests from blocked IPs
+    tcp-request connection silent-drop if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+
+    acl ip_high_conn_rate sc0_conn_rate gt 2
+
+    stick-table type ip size 10m expire 30m store gpc0,gpc1,conn_rate(1m),bytes_out_rate(10s),bytes_out_cnt	##about 500MB of memory
+    tcp-request connection track-sc0 src 						   #update table with src ip as key, store in sc0
+
+    #We Count rate-limit manualy -- Will be more CPU intensieve but will allow whitelists to enter and up to rateLimitSession non blacklisted IPs.
+    tcp-request connection track-sc1 int(1) table CONN_RATE
+
+    #tcp-request connection sc-set-gpt0(0) int(...) if ip_high_conn_rate is better but dies with:
+    #parsing [/usr/local/etc/haproxy/haproxy.cfg:53] : internal error, unexpected rule->from=0, please report this bug!
+    #<1> Mark Blacklist
+    tcp-request connection sc-inc-gpc0(0) if ip_high_conn_rate
+
+    #This connection is silently dropped no reason to count it for rateLimitSession
+    tcp-request connection sc-inc-gpc1(1) unless { sc0_get_gpc0() ge 1 }
+
+    # an IP that was blacklisted due to to many unsucsessfull tcp attempts
+    #-1- Enforece Blacklist
+    tcp-request connection silent-drop if { sc0_get_gpc0() ge 1 }
+
+    #an IP that had a sucessfull connection.
+    #-2- Allow Whitelist
+    tcp-request connection accept if { sc0_get_gpc1() ge 1 }
+
+    #-3- Enforece RateLimit. Connection attempts by *new* IPs/sec
+    tcp-request connection reject if { sc1_gpc1_rate(CONN_RATE) gt  256 } #rateLimitSession
+
+    # This is a successfull connection i.e., was sent more than 16K bytes in the last 30 min
+    #tcp-request session sc-set-gpt0(0) int(...)  if { sc0_kbytes_out gt 16 }
+    #<2> Mark Whitelist
+    tcp-request session sc-inc-gpc1(0) if { sc0_kbytes_out gt 4 }
+
+    # -4- Break a long high rate connection
+    # maxBytesOutRate10sec: 100mb/s for 10 sec
+    tcp-request session reject if { sc0_bytes_out_rate gt 134217728 }
+
+backend validator-fn
+    default-server maxconn 16
+    server validator validator:6181
+
+#CONNRATE holds only entry with key 1: used for determening global conn rate
+backend CONN_RATE
+    stick-table type integer size 1 expire 10m store gpc1,gpc1_rate(1s)
+
+##################  HTTP: metrics & API
+defaults
+	mode http
+        retries 3
+        timeout queue 5s  #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/
+        timeout connect 5s
+        timeout server 60s #what makes sense? for silence between nodes?
+        timeout client 60s
+
+        timeout client-fin 3s #How long to hold an interrupted client connection.
+        timeout server-fin 1s
+
+	timeout http-request 60s #len of http request
+	timeout http-keep-alive 2s
+
+        rate-limit sessions 256
+
+frontend validator-metrics
+    mode http
+    option httplog
+    bind :9101
+    default_backend validator-metrics
+
+    # Deny requests from blocked IPs
+    tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+    http-request add-header Forwarded "for=%ci"
+
+backend validator-metrics
+    mode http
+    default-server maxconn 1024
+    server validator validator:9101
+
+frontend validator-api
+    mode http
+    option httplog
+    bind :8180
+    default_backend validator-api
+
+    # Deny requests from blocked IPs
+    tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+    http-request add-header Forwarded "for=%ci"
+
+backend validator-api
+    mode http
+    default-server maxconn 1024
+    server validator validator:8080