Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Docker] Adding HAProxy to docker compose
- Loading branch information
1 parent
74b8581
commit 300206b
Showing
3 changed files
with
204 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
# Add Blocked Ips here |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,180 @@ | ||
global | ||
log stdout len 10240 format raw local0 | ||
|
||
# Config manual: https://cbonte.github.io/haproxy-dconv/2.5/configuration.html | ||
# magic values : terraform/helm/aptos-node/values.yaml | ||
|
||
maxconn 1024 | ||
# This limits the whole HA Proxy impacting both validators and other frontends | ||
# maxconnrate 128 | ||
nbthread 4 | ||
|
||
#4MB for client facing sndbuf/rcvbuf. -- 100Mb/s with 300 mili latency (e.g., us-asia) | ||
tune.sndbuf.client 4194304 #tcpBufSize | ||
tune.rcvbuf.client 4194304 #tcpBufSize | ||
|
||
user nobody | ||
|
||
## TCP port defaults | ||
defaults | ||
log global | ||
mode tcp | ||
#option tcplog | ||
option dontlog-normal | ||
log-format "%ci:%cp - %sp[%rt] [%t] %ft %Tw/%Tc/%Tt %B [%ts] %ac/%fc/%bc/%sc/%rc %sq/%bq" | ||
maxconn 1024 #Validator network mesh + FN x2 | ||
retries 3 | ||
timeout queue 5s #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/ | ||
timeout connect 5s | ||
# enough for 1 successfull + 5 unsuccessfull HB(10 sec interval) + 20 sec timeout | ||
timeout server 80s | ||
timeout client 80s | ||
|
||
timeout client-fin 3s #How long to hold an interrupted client connection. | ||
timeout server-fin 1s | ||
|
||
frontend fe-validator | ||
bind :6180 | ||
default_backend validator | ||
|
||
# Deny requests from blocked IPs | ||
tcp-request connection silent-drop if { src -n -f /usr/local/etc/haproxy/blocked.ips } | ||
|
||
# We deem a connection rate high when an IP is attempting to reconnect more than twice a min | ||
acl ip_high_conn_rate sc0_conn_rate gt 2 | ||
|
||
stick-table type ip size 10m expire 30m store gpc0,gpc1,conn_rate(1m),bytes_out_rate(10s),bytes_out_cnt ##about 500MB of memory | ||
tcp-request connection track-sc0 src #update table with src ip as key, store in sc0 | ||
|
||
#We Count rate-limit manualy -- Will be more CPU intensieve but will allow whitelists to enter and up to rateLimitSession non blacklisted IPs. | ||
tcp-request connection track-sc1 int(1) table CONN_RATE | ||
|
||
#tcp-request connection sc-set-gpt0(0) int(...) if ip_high_conn_rate is better but dies with: | ||
#parsing [/usr/local/etc/haproxy/haproxy.cfg:53] : internal error, unexpected rule->from=0, please report this bug! | ||
#<1> Mark Blacklist | ||
tcp-request connection sc-inc-gpc0(0) if ip_high_conn_rate | ||
|
||
#This connection is silently dropped no reason to count it for rateLimitSession | ||
tcp-request connection sc-inc-gpc1(1) unless { sc0_get_gpc0() ge 1 } | ||
|
||
# an IP that was blacklisted due to to many unsucsessfull tcp attempts | ||
#-1- Enforece Blacklist | ||
tcp-request connection silent-drop if { sc0_get_gpc0() ge 1 } | ||
|
||
#an IP that had a sucessfull connection. | ||
#-2- Allow Whitelist | ||
tcp-request connection accept if { sc0_get_gpc1() ge 1 } | ||
|
||
#-3- Enforece RateLimit. Connection attempts by *new* IPs/sec | ||
tcp-request connection reject if { sc1_gpc1_rate(CONN_RATE) gt 256 } #rateLimitSession | ||
|
||
# This is a successfull connection i.e., was sent more than 16K bytes in the last 30 min | ||
#tcp-request session sc-set-gpt0(0) int(...) if { sc0_kbytes_out gt 16 } | ||
#<2> Mark Whitelist | ||
tcp-request session sc-inc-gpc1(0) if { sc0_kbytes_out gt 4 } | ||
|
||
# -4- Break a long high rate connection | ||
# maxBytesOutRate10sec: 100mb/s for 10 sec | ||
tcp-request session reject if { sc0_bytes_out_rate gt 134217728 } | ||
|
||
backend validator | ||
default-server maxconn 1024 | ||
server validator validator:6180 | ||
|
||
frontend fe-fullnode | ||
bind :6181 | ||
default_backend validator-fn | ||
|
||
# Deny requests from blocked IPs | ||
tcp-request connection silent-drop if { src -n -f /usr/local/etc/haproxy/blocked.ips } | ||
|
||
acl ip_high_conn_rate sc0_conn_rate gt 2 | ||
|
||
stick-table type ip size 10m expire 30m store gpc0,gpc1,conn_rate(1m),bytes_out_rate(10s),bytes_out_cnt ##about 500MB of memory | ||
tcp-request connection track-sc0 src #update table with src ip as key, store in sc0 | ||
|
||
#We Count rate-limit manualy -- Will be more CPU intensieve but will allow whitelists to enter and up to rateLimitSession non blacklisted IPs. | ||
tcp-request connection track-sc1 int(1) table CONN_RATE | ||
|
||
#tcp-request connection sc-set-gpt0(0) int(...) if ip_high_conn_rate is better but dies with: | ||
#parsing [/usr/local/etc/haproxy/haproxy.cfg:53] : internal error, unexpected rule->from=0, please report this bug! | ||
#<1> Mark Blacklist | ||
tcp-request connection sc-inc-gpc0(0) if ip_high_conn_rate | ||
|
||
#This connection is silently dropped no reason to count it for rateLimitSession | ||
tcp-request connection sc-inc-gpc1(1) unless { sc0_get_gpc0() ge 1 } | ||
|
||
# an IP that was blacklisted due to to many unsucsessfull tcp attempts | ||
#-1- Enforece Blacklist | ||
tcp-request connection silent-drop if { sc0_get_gpc0() ge 1 } | ||
|
||
#an IP that had a sucessfull connection. | ||
#-2- Allow Whitelist | ||
tcp-request connection accept if { sc0_get_gpc1() ge 1 } | ||
|
||
#-3- Enforece RateLimit. Connection attempts by *new* IPs/sec | ||
tcp-request connection reject if { sc1_gpc1_rate(CONN_RATE) gt 256 } #rateLimitSession | ||
|
||
# This is a successfull connection i.e., was sent more than 16K bytes in the last 30 min | ||
#tcp-request session sc-set-gpt0(0) int(...) if { sc0_kbytes_out gt 16 } | ||
#<2> Mark Whitelist | ||
tcp-request session sc-inc-gpc1(0) if { sc0_kbytes_out gt 4 } | ||
|
||
# -4- Break a long high rate connection | ||
# maxBytesOutRate10sec: 100mb/s for 10 sec | ||
tcp-request session reject if { sc0_bytes_out_rate gt 134217728 } | ||
|
||
backend validator-fn | ||
default-server maxconn 16 | ||
server validator validator:6181 | ||
|
||
#CONNRATE holds only entry with key 1: used for determening global conn rate | ||
backend CONN_RATE | ||
stick-table type integer size 1 expire 10m store gpc1,gpc1_rate(1s) | ||
|
||
################## HTTP: metrics & API | ||
defaults | ||
mode http | ||
retries 3 | ||
timeout queue 5s #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/ | ||
timeout connect 5s | ||
timeout server 60s #what makes sense? for silence between nodes? | ||
timeout client 60s | ||
|
||
timeout client-fin 3s #How long to hold an interrupted client connection. | ||
timeout server-fin 1s | ||
|
||
timeout http-request 60s #len of http request | ||
timeout http-keep-alive 2s | ||
|
||
rate-limit sessions 256 | ||
|
||
frontend validator-metrics | ||
mode http | ||
option httplog | ||
bind :9101 | ||
default_backend validator-metrics | ||
|
||
# Deny requests from blocked IPs | ||
tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips } | ||
http-request add-header Forwarded "for=%ci" | ||
|
||
backend validator-metrics | ||
mode http | ||
default-server maxconn 1024 | ||
server validator validator:9101 | ||
|
||
frontend validator-api | ||
mode http | ||
option httplog | ||
bind :8180 | ||
default_backend validator-api | ||
|
||
# Deny requests from blocked IPs | ||
tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips } | ||
http-request add-header Forwarded "for=%ci" | ||
|
||
backend validator-api | ||
mode http | ||
default-server maxconn 1024 | ||
server validator validator:8080 |