Add backend health check from haproxy #1317

deepfence · Jul 19, 2023 · 0fb5319 · 0fb5319
1 parent 151d6d1
commit 0fb5319
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 17 deletions.
diff --git a/deepfence_backend/dockerify/haproxy/Dockerfile b/deepfence_backend/dockerify/haproxy/Dockerfile
@@ -29,7 +29,8 @@ ENV ENABLE_AUTH=true \
     FETCHER_SERVICE_HOST=deepfence-fetcher \
     FETCHER_SERVICE_PORT=8006 \
     CUSTOMER_UNIQUE_ID="" \
-    KAFKA_REST_PROXY=deepfence-rest-proxy:8082 \
+    KAFKA_REST_PROXY_PORT=8082 \
+    KAFKA_REST_PROXY=deepfence-rest-proxy:$KAFKA_REST_PROXY_PORT \
     FORCE_HTTPS_REDIRECT="true"
 
 COPY --from=minica-builder /go/minica.pem /usr/local/etc/haproxy/deepfence.crt

diff --git a/deepfence_backend/dockerify/haproxy/haproxy.cfg b/deepfence_backend/dockerify/haproxy/haproxy.cfg
@@ -29,6 +29,7 @@ defaults
     timeout connect 10s
     timeout server 60s
     timeout client 60s
+    timeout check 5s
     option dontlognull
     # option httplog
     log global
@@ -94,6 +95,12 @@ frontend port_443
 
 backend deepfence-ui-4042
     mode http
+
+    # health check
+    option httpchk
+    http-check send meth GET uri /health
+    http-check expect status 200
+
     acl is_dev_build str("${IS_DEV_BUILD}") -m str "true"
     # CORS
     http-response set-header Access-Control-Allow-Origin "*" if is_dev_build
@@ -117,10 +124,16 @@ backend deepfence-ui-4042
     # http-response set-header Referrer-Policy "strict-origin-when-cross-origin"
     # http-response set-header Content-Security-Policy "default-src 'self'"
     # http-response set-header X-Permitted-Cross-Domain-Policies "none"
-    server s1 ${UI_SERVICE_NAME}:${UI_SERVICE_PORT}
+    server s1 ${UI_SERVICE_NAME}:${UI_SERVICE_PORT} check inter 10s fall 3 rise 1
 
 backend deepfence-api-9998
     mode http
+
+    # health check
+    option httpchk
+    http-check send meth GET uri /deepfence/v1.5/ping
+    http-check expect status 200
+
     acl is_dev_build str("${IS_DEV_BUILD}") -m str "true"
     # CORS
     http-response set-header Access-Control-Allow-Origin "*" if is_dev_build
@@ -144,10 +157,16 @@ backend deepfence-api-9998
     # http-response set-header Referrer-Policy "strict-origin-when-cross-origin"
     # http-response set-header Content-Security-Policy "default-src 'self'"
     # http-response set-header X-Permitted-Cross-Domain-Policies "none"
-    server s1 ${API_SERVICE_HOST}:${API_SERVICE_PORT}
+    server s1 ${API_SERVICE_HOST}:${API_SERVICE_PORT} check inter 10s fall 3 rise 1
 
 backend deepfence-topology-8004
     mode http
+
+    # health check
+    option httpchk
+    http-check send meth GET uri /topology-api
+    http-check expect status 200
+
     http-request set-var(txn.deepfence_key) url_param(api_key)
     # http-request set-var(txn.mandatory_auth_verify) str(true)
     http-request lua.deepfence-key-verify
@@ -176,10 +195,16 @@ backend deepfence-topology-8004
     # http-response set-header Content-Security-Policy "default-src 'self'"
     # http-response set-header X-Permitted-Cross-Domain-Policies "none"
     # http-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization, deepfence-key, x-hosts, cache-control"
-    server s1 ${TOPOLOGY_SERVICE_HOST}:${TOPOLOGY_SERVICE_PORT}
+    server s1 ${TOPOLOGY_SERVICE_HOST}:${TOPOLOGY_SERVICE_PORT} check inter 10s fall 3 rise 1
 
 backend vulnerability-mapper-8001
     mode http
+
+    # health check
+    option httpchk
+    http-check send meth GET uri /vulnerability-mapper-api/ping
+    http-check expect status 200
+
     # http-request set-var(txn.mandatory_auth_verify) str(true)
     http-request lua.deepfence-key-verify
     http-request deny unless { var(txn.authorized) eq "true" } || { var(txn.authorized) eq "not_enabled" }
@@ -192,10 +217,16 @@ backend vulnerability-mapper-8001
     http-response set-header Cross-Origin-Resource-Policy "same-origin" unless is_dev_build
     http-response set-header Cross-Origin-Opener-Policy "same-origin" unless is_dev_build
     http-response set-header Cross-Origin-Embedder-Policy "require-corp" unless is_dev_build
-    server s1 ${VULNERABILITY_MAPPER_SERVICE_HOST}:${VULNERABILITY_MAPPER_SERVICE_PORT}
+    server s1 ${VULNERABILITY_MAPPER_SERVICE_HOST}:${VULNERABILITY_MAPPER_SERVICE_PORT} check inter 10s fall 3 rise 1
 
 backend fetcher-api-8006
     mode http
+
+    # health check
+    option httpchk
+    http-check send meth GET uri /health
+    http-check expect status 200
+
     # http-request set-var(txn.mandatory_auth_verify) str(true)
     http-request lua.deepfence-key-verify
     http-request deny unless { var(txn.authorized) eq "true" } || { var(txn.authorized) eq "not_enabled" }
@@ -222,7 +253,7 @@ backend fetcher-api-8006
     # http-response set-header Content-Security-Policy "default-src 'self'"
     # http-response set-header X-Permitted-Cross-Domain-Policies "none"
     # http-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization, deepfence-key, x-hosts, cache-control"
-    server s1 ${FETCHER_SERVICE_HOST}:${FETCHER_SERVICE_PORT}
+    server s1 ${FETCHER_SERVICE_HOST}:${FETCHER_SERVICE_PORT} check port ${FETCHER_SERVICE_PORT} inter 10s fall 3 rise 1
 
 backend deepfence-rest-proxy-8082
     mode http
@@ -232,7 +263,7 @@ backend deepfence-rest-proxy-8082
     http-request replace-path /ingest(.*) \1
     acl is_default_customer_id str("${CUSTOMER_UNIQUE_ID}") -m len 0
     http-request set-path %[path]-"${CUSTOMER_UNIQUE_ID}" if !is_default_customer_id
-    server s1 ${KAFKA_REST_PROXY}
+    server s1 ${KAFKA_REST_PROXY} check port ${KAFKA_REST_PROXY_PORT} inter 10s fall 3 rise 1
 
 backend options_method_handler
     mode http

diff --git a/deepfence_console/fetcher/fetcher-server.go b/deepfence_console/fetcher/fetcher-server.go
@@ -1495,6 +1495,11 @@ func main() {
 	httpMux.HandleFunc("/vulnerability-db/listing.json", vulnerabilityDbListing)
 	httpMux.HandleFunc("/df-api/upload-vulnerability-db", handleVulnerabilityFeedTarUpload)
 
+	// Health Check
+	httpMux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("{\"status\":\"Healthy\"}"))
+	})
 	fmt.Println("fetcher server is starting")
 
 	logger := log.New(os.Stdout, "fetcher-server: ", log.LstdFlags)

diff --git a/deepfence_ui/server.js b/deepfence_ui/server.js
@@ -35,6 +35,11 @@ backendProxy.on('error', err => console.error('Proxy error', err));
 app.all('/api*', backendProxy.web.bind(backendProxy));
 app.all('/ws*', backendProxy.web.bind(backendProxy));
 
+// Set health check url
+app.get('/health', (req, res) => {
+  res.writeHead(200, {'Content-Type': 'application/json'});
+  return res.end('{"status":"Healthy"}');
+})
 /**
  *
  * Production env serves precompiled content from build/

diff --git a/deployment-scripts/docker-compose.yml b/deployment-scripts/docker-compose.yml
@@ -9,7 +9,7 @@ services:
       core: 0
     network_mode: "host"
     pid: "host"
-    restart: "no"
+    restart: "always"
     logging:
       driver: "json-file"
       options:
@@ -20,7 +20,7 @@ services:
     image: ${IMAGE_REPOSITORY:-deepfenceio}/deepfence_postgres_ce:${DF_IMG_TAG:-1.5.0}
     ulimits:
       core: 0
-    restart: on-failure
+    restart: always
     networks:
       - deepfence_net
     volumes:
@@ -37,7 +37,7 @@ services:
       INITIALIZE_REDIS: "Y"
     ulimits:
       core: 0
-    restart: on-failure
+    restart: always
     networks:
       - deepfence_net
     depends_on:
@@ -56,7 +56,7 @@ services:
       - deepfence-postgres
     networks:
       - deepfence_net
-    restart: on-failure
+    restart: always
     volumes:
       - deepfence_data:/data:rw
     logging:
@@ -73,7 +73,7 @@ services:
       - deepfence_net
     ports:
       - "127.0.0.1:8004:8004"
-    restart: on-failure
+    restart: always
     command: topology
     logging:
       driver: "json-file"
@@ -89,7 +89,7 @@ services:
       resources:
         limits:
           cpus: ".2"
-    restart: on-failure
+    restart: always
     environment:
     - MGMT_CONSOLE_URL=127.0.0.1
     - MGMT_CONSOLE_PORT=8443
@@ -110,7 +110,7 @@ services:
   deepfence-es:
     container_name: deepfence-es-master
     image: ${IMAGE_REPOSITORY:-deepfenceio}/deepfence_elastic_ce:${DF_IMG_TAG:-1.5.0}
-    restart: on-failure
+    restart: always
     networks:
       - deepfence_net
     volumes:
@@ -260,6 +260,7 @@ services:
       core: 0
     tmpfs:
       - /tmp
+    restart: always
     environment:
       ELASTICSEARCH_SCHEME: "http"
     networks:
@@ -284,7 +285,7 @@ services:
     ulimits:
       core: 0
     entrypoint: /app/code/dockerify/celery/entrypoint.sh
-    restart: on-failure
+    restart: always
     networks:
       - deepfence_net
     depends_on:
@@ -303,7 +304,7 @@ services:
     ulimits:
       core: 0
     entrypoint: /app/code/dockerify/backend/entrypoint.sh
-    restart: on-failure
+    restart: always
     environment:
       - DEBUG=false
       - KAFKA_BROKERS=deepfence-kafka-broker:9092
@@ -328,7 +329,7 @@ services:
       core: 0
     networks:
       - deepfence_net
-    restart: on-failure
+    restart: always
     depends_on:
       - deepfence-topology
     logging:
@@ -417,6 +418,7 @@ services:
     image: ${IMAGE_REPOSITORY:-deepfenceio}/deepfence_kafka_broker_ce:${DF_IMG_TAG:-1.5.0}
     hostname: deepfence-kafka-broker
     container_name: deepfence-kafka-broker
+    restart: always
     ports:
       - "127.0.0.1:9092:9092"
       - "127.0.0.1:9101:9101"
@@ -437,6 +439,7 @@ services:
       - "127.0.0.1:8082:8082"
     hostname: deepfence-rest-proxy
     container_name: deepfence-rest-proxy
+    restart: always
     environment:
       - KAFKA_REST_BOOTSTRAP_SERVERS=deepfence-kafka-broker:9092
     networks: