Adds content-app to status API

A new model is introduced called `ContentAppStatus` is introduced which
will record status check-in records from the Content App.

The Content App will check in periodically checkin and write to the
`ContentAppStatus` object.

To tell the ContentAppStatus records apart for multiple Content Apps
each Content App now requires a name. The name is auto-selected as
'{pid}@{hostname}'.

The Status API viewset now shows a 'content_apps' section which lists
the ContentAppStatus records, showing their names and last heartbeat
time.

Adds a doc and feature changelog entries.

https://pulp.plan.io/issues/4881
closes #4881

CHANGES/4881.doc

@@ -0,0 +1 @@
+Adds documentation about the ``CONTENT_APP_TTL`` setting to the configuration page.

CHANGES/4881.feature

@@ -0,0 +1,4 @@
+Each Content App now heartbeats periodically, and Content Apps with recent heartbeats are shown in
+the Status API ``/pulp/api/v3/status/`` as a list called ``online_content_apps``. A new setting is
+introduced named ``CONTENT_APP_TTL`` which specifies the maximum time (in seconds) a Content App can
+not heartbeat and be considered online.

docs/installation/configuration.rst

@@ -210,6 +210,16 @@ CONTENT_PATH_PREFIX
    Defaults to ``'/pulp/content/'``.
 
 
+.. _content-app-ttl:
+
+CONTENT_APP_TTL
+^^^^^^^^^^^^^^^
+
+   The number of seconds before a content app should be considered lost.
+
+   Defaults to ``30`` seconds.
+
+
 .. _remote-user-environ-name:
 
 REMOTE_USER_ENVIRON_NAME

pulpcore/app/migrations/0002_contentappstatus.py

@@ -0,0 +1,27 @@
+# Generated by Django 2.2.2 on 2019-06-22 21:13
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ContentAppStatus',
+            fields=[
+                ('_id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('_created', models.DateTimeField(auto_now_add=True)),
+                ('_last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('name', models.CharField(db_index=True, max_length=255, unique=True)),
+                ('last_heartbeat', models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

pulpcore/app/models/__init__.py

@@ -21,6 +21,9 @@
     RepositoryVersion,
     RepositoryVersionContentDetails,
 )
+
+from .status import ContentAppStatus  # noqa
+
 from .task import CreatedResource, ReservedResource, Task, TaskReservedResource, Worker  # noqa
 
 # Moved here to avoid a circular import with Task

pulpcore/app/models/status.py

@@ -0,0 +1,88 @@
+"""
+Django models related to the Status API
+"""
+from datetime import timedelta
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+
+from pulpcore.app.models import Model
+
+
+class ContentAppStatusManager(models.Manager):
+
+    def online(self):
+        """
+        Returns a queryset of ``ContentAppStatus`` objects that are online.
+
+        To be considered 'online', a ContentAppStatus must have a heartbeat timestamp within
+        ``settings.CONTENT_APP_TTL`` from now.
+
+        Returns:
+            :class:`django.db.models.query.QuerySet`:  A query set of the ``ContentAppStatus``
+                objects which are considered 'online'.
+        """
+        now = timezone.now()
+        age_threshold = now - timedelta(seconds=settings.CONTENT_APP_TTL)
+
+        return self.filter(last_heartbeat__gte=age_threshold)
+
+
+class ContentAppStatus(Model):
+    """
+    Represents a Content App Status
+
+    Fields:
+
+        name (models.CharField): The name of the content app
+        last_heartbeat (models.DateTimeField): A timestamp of this worker's last heartbeat
+    """
+    objects = ContentAppStatusManager()
+
+    name = models.CharField(db_index=True, unique=True, max_length=255)
+    last_heartbeat = models.DateTimeField(auto_now=True)
+
+    @property
+    def online(self):
+        """
+        Whether a content app can be considered 'online'
+
+        To be considered 'online', a content app must have a heartbeat timestamp more recent than
+        the ``CONTENT_APP_TTL`` setting.
+
+        Returns:
+            bool: True if the content app is considered online, otherwise False
+        """
+        now = timezone.now()
+        age_threshold = now - timedelta(seconds=settings.CONTENT_APP_TTL)
+
+        return self.last_heartbeat >= age_threshold
+
+    @property
+    def missing(self):
+        """
+        Whether a Content App can be considered 'missing'
+
+        To be considered 'missing', a Content App must have a timestamp older than
+        ``SETTINGS.CONTENT_APP_TTL``.
+
+        Returns:
+            bool: True if the content app is considered missing, otherwise False
+        """
+        now = timezone.now()
+        age_threshold = now - timedelta(seconds=settings.CONTENT_APP_TTL)
+
+        return self.last_heartbeat < age_threshold
+
+    def save_heartbeat(self):
+        """
+        Update the last_heartbeat field to now and save it.
+
+        Only the last_heartbeat field will be saved. No other changes will be saved.
+
+        Raises:
+            ValueError: When the model instance has never been saved before. This method can
+                only update an existing database record.
+        """
+        self.save(update_fields=['last_heartbeat'])

pulpcore/app/serializers/status.py

@@ -2,7 +2,7 @@
 
 from rest_framework import serializers
 
-from pulpcore.app.serializers.task import WorkerSerializer
+from pulpcore.app.serializers.task import ContentAppStatusSerializer, WorkerSerializer
 
 
 class VersionSerializer(serializers.Serializer):
@@ -62,6 +62,12 @@ class StatusSerializer(serializers.Serializer):
         many=True
     )
 
+    online_content_apps = ContentAppStatusSerializer(
+        help_text=_("List of online content apps known to the application. An online worker is "
+                    "actively heartbeating"),
+        many=True
+    )
+
     database_connection = DatabaseConnectionSerializer(
         help_text=_("Database connection information")
     )

pulpcore/app/serializers/task.py

@@ -115,6 +115,21 @@ class Meta:
         fields = ('state',)
 
 
+class ContentAppStatusSerializer(ModelSerializer):
+    name = serializers.CharField(
+        help_text=_('The name of the worker.'),
+        read_only=True
+    )
+    last_heartbeat = serializers.DateTimeField(
+        help_text=_('Timestamp of the last time the worker talked to the service.'),
+        read_only=True
+    )
+
+    class Meta:
+        model = models.ContentAppStatus
+        fields = ('name', 'last_heartbeat')
+
+
 class WorkerSerializer(ModelSerializer):
     _href = IdentityField(view_name='workers-detail')
 

pulpcore/app/settings.py

@@ -204,6 +204,7 @@
 
 CONTENT_HOST = None
 CONTENT_PATH_PREFIX = '/pulp/content/'
+CONTENT_APP_TTL = 30
 
 REMOTE_USER_ENVIRON_NAME = "REMOTE_USER"
 

pulpcore/app/views/status.py

@@ -6,6 +6,7 @@
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
+from pulpcore.app.models.status import ContentAppStatus
 from pulpcore.app.models.task import Worker
 from pulpcore.app.serializers.status import StatusSerializer
 from pulpcore.app.settings import INSTALLED_PULP_PLUGINS
@@ -49,10 +50,16 @@ def get(self, request, format=None):
         except Exception:
             missing_workers = None
 
+        try:
+            online_content_apps = ContentAppStatus.objects.online()
+        except Exception:
+            online_content_apps = None
+
         data = {
             'versions': versions,
             'online_workers': online_workers,
             'missing_workers': missing_workers,
+            'online_content_apps': online_content_apps,
             'database_connection': db_status,
             'redis_connection': redis_status
         }

pulpcore/content/__init__.py

@@ -1,19 +1,46 @@
+import asyncio
 from contextlib import suppress
+from gettext import gettext as _
 from importlib import import_module
+import logging
+import os
+import socket
+
+import django  # noqa otherwise E402: module level not at top of file
+django.setup()  # noqa otherwise E402: module level not at top of file
 
 from aiohttp import web
 from django.conf import settings
 
 from pulpcore.app.apps import pulp_plugin_configs
+from pulpcore.app.models import ContentAppStatus
 
 from .handler import Handler
 
+
+log = logging.getLogger(__name__)
+
 app = web.Application()
 
 CONTENT_MODULE_NAME = 'content'
 
 
+async def _heartbeat():
+    name = '{pid}@{hostname}'.format(pid=os.getpid(), hostname=socket.gethostname())
+    heartbeat_interval = settings.CONTENT_APP_TTL // 4
+    i8ln_msg = _("Content App '{name}' heartbeat written, sleeping for '{interarrival}' seconds")
+    msg = i8ln_msg.format(name=name, interarrival=heartbeat_interval)
+
+    while True:
+        content_app_status, created = ContentAppStatus.objects.get_or_create(name=name)
+        if not created:
+            content_app_status.save_heartbeat()
+        log.debug(msg)
+        await asyncio.sleep(heartbeat_interval)
+
+
 async def server(*args, **kwargs):
+    asyncio.ensure_future(_heartbeat())
     for pulp_plugin in pulp_plugin_configs():
         if pulp_plugin.name != "pulpcore.app":
             content_module_name = '{name}.{module}'.format(name=pulp_plugin.name,