Deployment: Keycloakx Cluster revise HA-Proxy configuration

- Fix errors during startup due to ha-proxy config - Remove unnecessary JVM Options - Move inifinispan-lb config to dedicated config files - Mark stickiness cookie as httponly, secure, samesite=none
thomasdarimont · May 22, 2023 · d1d4d00 · d1d4d00
1 parent 669f04d
commit d1d4d00
Show file tree

Hide file tree

Showing 7 changed files with 253 additions and 25 deletions.
diff --git a/deployments/local/clusterx/docker-compose.yml b/deployments/local/clusterx/docker-compose.yml
@@ -30,7 +30,7 @@ services:
 
 # Default JAVA_OPTS
       #JAVA_OPTS: -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true
-      JAVA_OPTS: "-XX:MaxRAMPercentage=80 -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+DisableExplicitGC -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -Djava.net.preferIPv4Stack=true"
+      JAVA_OPTS: "-XX:MaxRAMPercentage=80 -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+DisableExplicitGC -Djava.net.preferIPv4Stack=true"
 
       # Allow access via visualvm and jmc
       JAVA_TOOL_OPTIONS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=8790 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"

diff --git a/deployments/local/clusterx/haproxy-database-ispn/haproxy.cfg b/deployments/local/clusterx/haproxy-database-ispn/haproxy.cfg
@@ -49,9 +49,7 @@ global
 # use if not designated in their block
 #---------------------------------------------------------------------
 defaults
-    mode                    http
     log                     global
-    option                  httplog
     option                  dontlognull
     option http-server-close
     option forwardfor       except 127.0.0.0/8
@@ -68,9 +66,12 @@ defaults
     maxconn                 3000
 
 frontend id.acme.test
+    mode http
+    option httplog
     # Copy the haproxy.crt.pem file to /etc/haproxy
     bind *:1443 ssl crt /etc/haproxy/haproxy.crt.pem
 
+
     # ACLs based on typical "scanner noise"
     acl is_bad_url path -m end -i .php
     acl is_bad_url path -m end -i .asp
@@ -92,7 +93,7 @@ backend keycloak
     http-request add-header X-Forwarded-Port 1443
     http-request redirect scheme https unless { ssl_fc }
 
-    cookie KC_ROUTE insert indirect nocache
+    cookie KC_ROUTE insert indirect nocache secure httponly attr samesite=none
     balance roundrobin
 
 # Configure transport encryption with https / tls

diff --git a/...l/clusterx/haproxy-external-ispn-database/docker-compose-haproxy-ispn-remote-database.yml b/...l/clusterx/haproxy-external-ispn-database/docker-compose-haproxy-ispn-remote-database.yml
@@ -147,7 +147,7 @@ services:
           - infinispan-lb
     volumes:
       # relative paths needs to be relative to the docker-compose cwd.
-      - ../haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:z
+      - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:z
     #      - ../../../../config/stage/dev/tls/acme.test+1.pem:/etc/haproxy/haproxy.crt.pem:z
     #      - ../../../../config/stage/dev/tls/acme.test+1-key.pem:/etc/haproxy/haproxy.crt.pem.key:z
     # - ../run/haproxy/run:/var/run:z

diff --git a/deployments/local/clusterx/haproxy-external-ispn-database/haproxy.cfg b/deployments/local/clusterx/haproxy-external-ispn-database/haproxy.cfg
@@ -0,0 +1,121 @@
+# See https://www.haproxy.com/blog/the-four-essential-sections-of-an-haproxy-configuration/
+#---------------------------------------------------------------------
+# Global settings
+#---------------------------------------------------------------------
+global
+    # to have these messages end up in /var/log/haproxy.log you will
+    # need to:
+    #
+    # 1) configure syslog to accept network log events.  This is done
+    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
+    #    /etc/sysconfig/syslog
+    #
+    # 2) configure local2 events to go to the /var/log/haproxy.log
+    #   file. A line like the following can be added to
+    #   /etc/sysconfig/syslog
+    #
+    #    local2.*                       /var/log/haproxy.log
+    #
+    log         127.0.0.1 local2
+
+    #chroot      /var/lib/haproxy
+    #pidfile     /var/run/haproxy.pid
+    maxconn     4000
+    user        haproxy
+    group       haproxy
+    daemon
+
+    # turn on stats unix socket
+#    stats socket /var/lib/haproxy/stats
+
+    # utilize system-wide crypto-policies
+## Disable cipher config to workaround
+## Proxy 'id.acme.test': unable to set SSL cipher list to 'PROFILE=SYSTEM' for bind '*:1443' at [/usr/local/etc/haproxy/haproxy.cfg:58]
+#    ssl-default-bind-ciphers PROFILE=SYSTEM
+#    ssl-default-server-ciphers PROFILE=SYSTEM
+
+    # modern configuration
+    # generated via https://ssl-config.mozilla.org/#server=haproxy&version=2.1&config=modern&openssl=1.1.1d&ocsp=false&guideline=5.6
+    ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+    ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tlsv12 no-tls-tickets
+
+    ssl-default-server-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+
+    # Note that we left out no-tlsv12, since Keycloak currently uses tlsv12
+    ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
+
+#---------------------------------------------------------------------
+# common defaults that all the 'listen' and 'backend' sections will
+# use if not designated in their block
+#---------------------------------------------------------------------
+defaults
+    log                     global
+    option                  dontlognull
+    option http-server-close
+    option forwardfor       except 127.0.0.0/8
+    option                  redispatch
+    retries                 2
+# see https://cbonte.github.io/haproxy-dconv/2.4/configuration.html#3.9-timeout%20server
+    timeout http-request    10s
+    timeout queue           1m
+    timeout connect         2s
+    timeout client          1m
+    timeout server          1m
+    timeout http-keep-alive 10s
+    timeout check           3s
+    maxconn                 3000
+
+frontend id.acme.test
+    mode http
+    option httplog
+
+    # Copy the haproxy.crt.pem file to /etc/haproxy
+    bind *:1443 ssl crt /etc/haproxy/haproxy.crt.pem
+
+    # ACLs based on typical "scanner noise"
+    acl is_bad_url path -m end -i .php
+    acl is_bad_url path -m end -i .asp
+#    acl is_bad_url url  -m sub    ../..
+
+    # If the request matches one of the known "bad stuff" rules, reject.
+    http-request deny if is_bad_url
+
+    use_backend keycloak
+
+backend keycloak
+    mode http
+    stats enable
+    stats uri /haproxy?status
+    option httpchk
+    option forwardfor
+    http-check send meth GET uri /auth/realms/master ver HTTP/1.1 hdr Host localhost
+    http-request add-header X-Forwarded-Proto https
+    http-request add-header X-Forwarded-Port 1443
+    http-request redirect scheme https unless { ssl_fc }
+
+    cookie KC_ROUTE insert indirect nocache secure httponly attr samesite=none
+    balance roundrobin
+
+# Configure transport encryption with https / tls
+# http://cbonte.github.io/haproxy-dconv/2.4/configuration.html#check
+    server kc1 acme-keycloak-1:8443/auth ssl verify none check inter 2s downinter 1s fall 4 rise 3 cookie kc1
+    server kc2 acme-keycloak-2:8443/auth ssl verify none check inter 2s downinter 1s fall 4 rise 3 cookie kc2
+
+# Configure plain transport with http
+#    server kc1 acme-keycloak-1:8080/auth check cookie kc1
+#    server kc2 acme-keycloak-2:8080/auth check cookie kc2
+
+frontend infinispan-lb
+    mode tcp
+    bind *:11222
+    use_backend infinispan
+
+backend infinispan
+    mode tcp
+
+#    option httpchk
+#    http-check send meth GET uri /console/welcome ver HTTP/1.1 hdr Host localhost
+
+    balance roundrobin
+    server ispn1 acme-ispn-1:11222 check inter 2s downinter 1s fall 4 rise 3
+    server ispn2 acme-ispn-2:11222 check inter 2s downinter 1s fall 4 rise 3
diff --git a/deployments/local/clusterx/haproxy-external-ispn/docker-compose-haproxy-ispn-remote.yml b/deployments/local/clusterx/haproxy-external-ispn/docker-compose-haproxy-ispn-remote.yml
@@ -115,7 +115,7 @@ services:
     build: ../haproxy
     volumes:
       # relative paths needs to be relative to the docker-compose cwd.
-      - ../haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:z
+      - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:z
     #      - ../../../../config/stage/dev/tls/acme.test+1.pem:/etc/haproxy/haproxy.crt.pem:z
     #      - ../../../../config/stage/dev/tls/acme.test+1-key.pem:/etc/haproxy/haproxy.crt.pem.key:z
     # - ../run/haproxy/run:/var/run:z

diff --git a/deployments/local/clusterx/haproxy-external-ispn/haproxy.cfg b/deployments/local/clusterx/haproxy-external-ispn/haproxy.cfg
@@ -0,0 +1,122 @@
+# See https://www.haproxy.com/blog/the-four-essential-sections-of-an-haproxy-configuration/
+#---------------------------------------------------------------------
+# Global settings
+#---------------------------------------------------------------------
+global
+    # to have these messages end up in /var/log/haproxy.log you will
+    # need to:
+    #
+    # 1) configure syslog to accept network log events.  This is done
+    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
+    #    /etc/sysconfig/syslog
+    #
+    # 2) configure local2 events to go to the /var/log/haproxy.log
+    #   file. A line like the following can be added to
+    #   /etc/sysconfig/syslog
+    #
+    #    local2.*                       /var/log/haproxy.log
+    #
+    log         127.0.0.1 local2
+
+    #chroot      /var/lib/haproxy
+    #pidfile     /var/run/haproxy.pid
+    maxconn     4000
+    user        haproxy
+    group       haproxy
+    daemon
+
+    # turn on stats unix socket
+#    stats socket /var/lib/haproxy/stats
+
+    # utilize system-wide crypto-policies
+## Disable cipher config to workaround
+## Proxy 'id.acme.test': unable to set SSL cipher list to 'PROFILE=SYSTEM' for bind '*:1443' at [/usr/local/etc/haproxy/haproxy.cfg:58]
+#    ssl-default-bind-ciphers PROFILE=SYSTEM
+#    ssl-default-server-ciphers PROFILE=SYSTEM
+
+    # modern configuration
+    # generated via https://ssl-config.mozilla.org/#server=haproxy&version=2.1&config=modern&openssl=1.1.1d&ocsp=false&guideline=5.6
+    ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+    ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tlsv12 no-tls-tickets
+
+    ssl-default-server-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+
+    # Note that we left out no-tlsv12, since Keycloak currently uses tlsv12
+    ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
+
+#---------------------------------------------------------------------
+# common defaults that all the 'listen' and 'backend' sections will
+# use if not designated in their block
+#---------------------------------------------------------------------
+defaults
+#    mode                    http
+    log                     global
+    option                  dontlognull
+    option http-server-close
+    option forwardfor       except 127.0.0.0/8
+    option                  redispatch
+    retries                 2
+# see https://cbonte.github.io/haproxy-dconv/2.4/configuration.html#3.9-timeout%20server
+    timeout http-request    10s
+    timeout queue           1m
+    timeout connect         2s
+    timeout client          1m
+    timeout server          1m
+    timeout http-keep-alive 10s
+    timeout check           3s
+    maxconn                 3000
+
+frontend id.acme.test
+    mode http
+    option httplog
+
+    # Copy the haproxy.crt.pem file to /etc/haproxy
+    bind *:1443 ssl crt /etc/haproxy/haproxy.crt.pem
+
+    # ACLs based on typical "scanner noise"
+    acl is_bad_url path -m end -i .php
+    acl is_bad_url path -m end -i .asp
+#    acl is_bad_url url  -m sub    ../..
+
+    # If the request matches one of the known "bad stuff" rules, reject.
+    http-request deny if is_bad_url
+
+    use_backend keycloak
+
+backend keycloak
+    mode http
+    stats enable
+    stats uri /haproxy?status
+    option httpchk
+    option forwardfor
+    http-check send meth GET uri /auth/realms/master ver HTTP/1.1 hdr Host localhost
+    http-request add-header X-Forwarded-Proto https
+    http-request add-header X-Forwarded-Port 1443
+    http-request redirect scheme https unless { ssl_fc }
+
+    cookie KC_ROUTE insert indirect nocache secure httponly attr samesite=none
+    balance roundrobin
+
+# Configure transport encryption with https / tls
+# http://cbonte.github.io/haproxy-dconv/2.4/configuration.html#check
+    server kc1 acme-keycloak-1:8443/auth ssl verify none check inter 2s downinter 1s fall 4 rise 3 cookie kc1
+    server kc2 acme-keycloak-2:8443/auth ssl verify none check inter 2s downinter 1s fall 4 rise 3 cookie kc2
+
+# Configure plain transport with http
+#    server kc1 acme-keycloak-1:8080/auth check cookie kc1
+#    server kc2 acme-keycloak-2:8080/auth check cookie kc2
+
+frontend infinispan-lb
+    mode tcp
+    bind *:11222
+    use_backend infinispan
+
+backend infinispan
+    mode tcp
+
+#    option httpchk
+#    http-check send meth GET uri /console/welcome ver HTTP/1.1 hdr Host localhost
+
+    balance roundrobin
+    server ispn1 acme-ispn-1:11222 check inter 2s downinter 1s fall 4 rise 3
+    server ispn2 acme-ispn-2:11222 check inter 2s downinter 1s fall 4 rise 3
diff --git a/deployments/local/clusterx/haproxy/haproxy.cfg b/deployments/local/clusterx/haproxy/haproxy.cfg
@@ -49,9 +49,7 @@ global
 # use if not designated in their block
 #---------------------------------------------------------------------
 defaults
-#    mode                    http
     log                     global
-    option                  httplog
     option                  dontlognull
     option http-server-close
     option forwardfor       except 127.0.0.0/8
@@ -71,6 +69,7 @@ frontend id.acme.test
     mode http
     # Copy the haproxy.crt.pem file to /etc/haproxy
     bind *:1443 ssl crt /etc/haproxy/haproxy.crt.pem
+    option httplog
 
     # ACLs based on typical "scanner noise"
     acl is_bad_url path -m end -i .php
@@ -87,13 +86,13 @@ backend keycloak
     stats enable
     stats uri /haproxy?status
     option httpchk
-    http-check send meth GET uri /auth/realms/master ver HTTP/1.1 hdr Host localhost
     option forwardfor
+    http-check send meth GET uri /auth/realms/master ver HTTP/1.1 hdr Host localhost
     http-request add-header X-Forwarded-Proto https
     http-request add-header X-Forwarded-Port 1443
     http-request redirect scheme https unless { ssl_fc }
 
-    cookie KC_ROUTE insert indirect nocache
+    cookie KC_ROUTE insert indirect nocache secure httponly attr samesite=none
     balance roundrobin
 
 # Configure transport encryption with https / tls
@@ -104,18 +103,3 @@ backend keycloak
 # Configure plain transport with http
 #    server kc1 acme-keycloak-1:8080/auth check cookie kc1
 #    server kc2 acme-keycloak-2:8080/auth check cookie kc2
-
-frontend infinispan-lb
-    mode tcp
-    bind *:11222
-    use_backend infinispan
-
-backend infinispan
-    mode tcp
-
-#    option httpchk
-#    http-check send meth GET uri /console/welcome ver HTTP/1.1 hdr Host localhost
-
-    balance roundrobin
-    server ispn1 acme-ispn-1:11222 check inter 2s downinter 1s fall 4 rise 3
-    server ispn2 acme-ispn-2:11222 check inter 2s downinter 1s fall 4 rise 3