Add documentation about docker-compose/k8s health checks (#5449)

--------

Co-authored-by: Hanlin Miao <46973263+HanlinMiao@users.noreply.github.com>
Co-authored-by: Gary Snider <75227981+gsnider2195@users.noreply.github.com>

changes/5340.documentation

@@ -0,0 +1 @@
+Added installation documentation about recommended health-checks for Docker Compose and Kubernetes.

development/dev.env

@@ -28,6 +28,7 @@ POSTGRES_USER=nautobot
 
 # Needed for Redis, must match the values for Nautobot above
 REDIS_PASSWORD=decinablesprewad
+REDISCLI_AUTH=$REDIS_PASSWORD
 
 # Needed for Selenium integration tests
 # WebDriver (Selenium client)

development/docker-compose.mysql.yml

@@ -19,10 +19,10 @@ services:
       - ./mysql-unittests.sql:/docker-entrypoint-initdb.d/mysql-unittests.sql
       - mysqldata_nautobot:/var/lib/mysql
     healthcheck:
-      test: mysql -h localhost -u $$MYSQL_USER --password=$$MYSQL_PASSWORD --execute "SHOW DATABASES;"
-      start_period: 5s
-      interval: 5s
+      interval: 10s
       timeout: 5s
-      retries: 50
+      start_period: 30s
+      retries: 3
+      test: 'mysql -h localhost -u $$MYSQL_USER --password=$$MYSQL_PASSWORD --execute "SHOW DATABASES;"'
 volumes:
   mysqldata_nautobot:

development/docker-compose.postgres.yml

@@ -8,9 +8,10 @@ services:
     volumes:
       - pgdata_nautobot:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d $$POSTGRES_DB -U $$POSTGRES_USER"]
-      interval: 5s
+      interval: 10s
       timeout: 5s
-      retries: 50
+      start_period: 30s
+      retries: 3
+      test: "pg_isready -d $$POSTGRES_DB -U $$POSTGRES_USER"
 volumes:
   pgdata_nautobot:

development/docker-compose.yml

@@ -26,15 +26,11 @@ services:
       - dev.env
     tty: true
     healthcheck:
-      interval: 5s
-      timeout: 5s
+      interval: 10s
+      timeout: 10s
       start_period: 5m  # it takes a WHILE to run initial migrations with an empty DB
       retries: 3
-      test:
-        - "CMD"
-        - "curl"
-        - "-f"
-        - "http://localhost:8080/health/"
+      test: "nautobot-server health_check"
   celery_worker:
     image: "local/nautobot-dev:local-py${PYTHON_VER}"
     ports:
@@ -44,14 +40,10 @@ services:
     entrypoint: "watchmedo auto-restart --directory './' --pattern '*.py' --recursive -- nautobot-server celery worker -l INFO --events"
     healthcheck:
       interval: 60s
-      timeout: 30s
+      timeout: 10s
       start_period: 30s
       retries: 3
-      test:
-        - "CMD"
-        - "bash"
-        - "-c"
-        - "nautobot-server celery inspect ping --destination celery@$$HOSTNAME"
+      test: "nautobot-server celery inspect ping --destination celery@$$HOSTNAME"
     depends_on:
       nautobot:
         condition: service_healthy
@@ -68,12 +60,8 @@ services:
       timeout: 5s
       start_period: 30s
       retries: 3
-      test:
-        - "CMD"
-        - "/bin/sh"
-        - "-c"
-        # find the heartbeat file and report success if it was modified less than 0.1 minutes (6 seconds) ago, else fail
-        - '[ $$(find /tmp/nautobot_celery_beat_heartbeat -mmin -0.1 | wc -l) -eq 1 ] || false'
+      # find the heartbeat file and report success if it was modified less than 0.1 minutes (6 seconds) ago, else fail
+      test: "[ $$(find /tmp/nautobot_celery_beat_heartbeat -mmin -0.1 | wc -l) -eq 1 ] || false"
     depends_on:
       nautobot:
         condition: service_healthy
@@ -88,8 +76,13 @@ services:
       - sh
       - -c # this is to evaluate the $REDIS_PASSWORD from the env
       - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
+    healthcheck:
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      test: "redis-cli -h localhost ping | grep PONG"
     env_file:
-      - dev.env
+      - ./dev.env
   selenium:
     image: selenium/standalone-firefox:4.9.1
     ports:

mkdocs.yml

@@ -280,6 +280,7 @@ nav:
         - External Authentication (Optional): "user-guide/administration/installation/external-authentication.md"
         - Installing and Using Plugins: "user-guide/administration/installation/app-install.md"
         - Nautobot Docker Images: "user-guide/administration/installation/docker.md"
+        - Health Checks: "user-guide/administration/installation/health-checks.md"
         - SELinux Troubleshooting: "user-guide/administration/installation/selinux-troubleshooting.md"
       - Upgrading:
         - Database Backup: "user-guide/administration/upgrading/database-backup.md"

nautobot/docs/user-guide/administration/installation/docker.md

@@ -76,6 +76,9 @@ In addition to all tags described in the previous section, the following additio
 
 Nautobot requires a MySQL or PostgreSQL database and Redis instance before it will start. Because of this the quickest and easiest way to get Nautobot running is with [Docker Compose](https://docs.docker.com/compose/), which will install and configure PostgreSQL and Redis containers for you automatically.
 
+!!! tip
+    Whether you're using the Docker CLI, Docker Compose, or [Kubernetes](https://kubernetes.io/), in any case you'll want to set up appropriate [health checks](health-checks.md) for your containers.
+
 ## Configuration
 
 Most configuration parameters are available via environment variables which can be passed to the container. If you desire you can inject your own `nautobot_config.py` by overriding `/opt/nautobot/nautobot_config.py` using [docker volumes](https://docs.docker.com/storage/volumes/) by adding `-v /local/path/to/custom/nautobot_config.py:/opt/nautobot/nautobot_config.py` to your docker run command, for example: