Switch from curl-based default Docker healthcheck to a CLI-based one

changes/5342.added

@@ -0,0 +1 @@
+Added `MigrationsBackend` to health-check, which will fail if any unapplied database migrations are present.

changes/5342.changed

@@ -0,0 +1 @@
+Changed default Docker HEALTHCHECK to use `nautobot-server health_check` CLI command.

development/docker-compose.final.yml

@@ -8,12 +8,6 @@ services:
     image: "local/nautobot-final:local-py${PYTHON_VER}"
     ports:
       - 8443:8443
-    healthcheck:
-      test:
-        - "CMD"
-        - "curl"
-        - "-fk"
-        - "https://localhost:8443/health/"
   celery_worker:
     image: "local/nautobot-final:local-py${PYTHON_VER}"
     entrypoint: "nautobot-server celery worker -l INFO --events"

development/docker-compose.yml

@@ -25,16 +25,6 @@ services:
     env_file:
       - dev.env
     tty: true
-    healthcheck:
-      interval: 5s
-      timeout: 5s
-      start_period: 5m  # it takes a WHILE to run initial migrations with an empty DB
-      retries: 3
-      test:
-        - "CMD"
-        - "curl"
-        - "-f"
-        - "http://localhost:8080/health/"
   celery_worker:
     image: "local/nautobot-dev:local-py${PYTHON_VER}"
     ports:

docker/Dockerfile

@@ -58,7 +58,9 @@ RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \
     --mount=type=cache,target="/tmp",sharing=locked \
     pip install --upgrade pip wheel
 
-HEALTHCHECK --interval=5s --timeout=5s --start-period=5s --retries=1 CMD curl --fail http://localhost:8080/health/ || exit 1
+# timeout/interval=10s because `nautobot-server` can be slow to start - https://github.com/nautobot/nautobot/issues/4292
+# start-period=5m because initial migrations can take several minutes to run on a fresh DB
+HEALTHCHECK --interval=10s --timeout=10s --start-period=5m --retries=3 CMD nautobot-server health_check
 
 # Generate nautobot user and its required dirs for later consumption
 RUN mkdir /opt/nautobot /opt/nautobot/.cache /prom_cache /source && \

nautobot/core/settings.py

@@ -425,6 +425,10 @@
     "graphene_django",
     "health_check",
     "health_check.storage",
+    # We have custom implementations of these in nautobot.extras.health_checks:
+    # "health_check.db",
+    # "health_check.contrib.migrations",
+    # "health_check.contrib.redis",
     "django_extensions",
     "constance.backends.database",
     "django_ajax_tables",

nautobot/extras/apps.py

@@ -43,10 +43,11 @@ def convert_field_to_list_tags(field, registry=None):
                 "during the execution of the migration command for the first time."
             )
 
-        # Register the DatabaseBackend health check
-        from nautobot.extras.health_checks import DatabaseBackend, RedisBackend
+        # Register the DatabaseBackend, MigrationsBackend, and RedisBackend health checks
+        from nautobot.extras.health_checks import DatabaseBackend, MigrationsBackend, RedisBackend
 
         plugin_dir.register(DatabaseBackend)
+        plugin_dir.register(MigrationsBackend)
         plugin_dir.register(RedisBackend)
 
         # Register built-in SecretsProvider classes

nautobot/extras/health_checks.py

@@ -3,7 +3,8 @@
 from urllib.parse import urlparse
 
 from django.conf import settings
-from django.db import connection, DatabaseError, IntegrityError
+from django.db import connection, connections, DatabaseError, DEFAULT_DB_ALIAS, IntegrityError
+from django.db.migrations.executor import MigrationExecutor
 from health_check.backends import BaseHealthCheckBackend
 from health_check.exceptions import ServiceReturnedUnexpectedResult, ServiceUnavailable
 from prometheus_client import Gauge
@@ -65,10 +66,32 @@ def check_status(self):
                 obj.title = "newtest"
                 obj.save()
                 obj.delete()
-        except IntegrityError:
-            raise ServiceReturnedUnexpectedResult("Integrity Error")
-        except DatabaseError:
-            raise ServiceUnavailable("Database error")
+        except IntegrityError as e:
+            self.add_error(ServiceReturnedUnexpectedResult("Integrity Error"), e)
+        except DatabaseError as e:
+            self.add_error(ServiceUnavailable("Database error"), e)
+
+
+class MigrationsBackend(NautobotHealthCheckBackend):
+    """Check whether all migrations have been applied."""
+
+    metric = Gauge(
+        "health_check_migrations_info",
+        "State of migrations backend",
+        multiprocess_mode=NautobotHealthCheckBackend.MULTIPROCESS_MODE,
+    )
+
+    def check_status(self):
+        db_alias = getattr(settings, "HEALTHCHECK_MIGRATIONS_DB", DEFAULT_DB_ALIAS)
+        try:
+            executor = MigrationExecutor(connections[db_alias])
+            plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
+            if plan:
+                self.add_error(ServiceUnavailable("There are migrations not yet applied"))
+        except DatabaseError as e:
+            self.add_error(ServiceUnavailable("Database is not ready"), e)
+        except Exception as e:
+            self.add_error(ServiceUnavailable("Unknown error"), e)
 
 
 class RedisHealthCheck(NautobotHealthCheckBackend):