Update storage in status to be per domain for all storage types

fixes: pulp#4456
fixes: pulp#4457

CHANGES/4456.feature

@@ -0,0 +1 @@
+Status endpoint now reports storage usage of the domain being called from.

CHANGES/4457.feature

@@ -0,0 +1 @@
+Status endpoint now reports the storage usage for non-filesystem storage backends.

pulpcore/app/serializers/status.py

@@ -50,11 +50,17 @@ class StorageSerializer(serializers.Serializer):
     Serializer for information about the storage system
     """
 
-    total = serializers.IntegerField(min_value=0, help_text=_("Total number of bytes"))
+    total = serializers.IntegerField(
+        min_value=0, help_text=_("Total number of bytes"), allow_null=True
+    )
 
-    used = serializers.IntegerField(min_value=0, help_text=_("Number of bytes in use"))
+    used = serializers.IntegerField(
+        min_value=0, help_text=_("Number of bytes in use"), allow_null=True
+    )
 
-    free = serializers.IntegerField(min_value=0, help_text=_("Number of free bytes"))
+    free = serializers.IntegerField(
+        min_value=0, help_text=_("Number of free bytes"), allow_null=True
+    )
 
 
 class ContentSettingsSerializer(serializers.Serializer):

pulpcore/app/views/status.py

@@ -3,28 +3,37 @@
 from gettext import gettext as _
 
 from django.conf import settings
-from django.core.files.storage import default_storage
+from django.db.models import Sum
 from drf_spectacular.utils import extend_schema
 from rest_framework.response import Response
 from rest_framework.views import APIView
+from collections import namedtuple
 
 from pulpcore.app.apps import pulp_plugin_configs
+from pulpcore.app.models.content import Artifact
 from pulpcore.app.models.status import ApiAppStatus, ContentAppStatus
 from pulpcore.app.models.task import Worker
 from pulpcore.app.serializers.status import StatusSerializer
 from pulpcore.app.redis_connection import get_redis_connection
+from pulpcore.app.util import get_domain
 
 _logger = logging.getLogger(__name__)
+StorageSpace = namedtuple("StorageSpace", ("total", "used", "free"))
 
 
 def _disk_usage():
-    if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem":
+    domain = get_domain()
+    if domain.storage_class == "pulpcore.app.models.storage.FileSystem":
+        storage = domain.get_storage()
         try:
-            return shutil.disk_usage(default_storage.location)
+            return shutil.disk_usage(storage.location)
         except Exception:
             _logger.exception(_("Failed to determine disk usage"))
-
-    return None
+    else:
+        used = Artifact.objects.filter(pulp_domain=domain).aggregate(
+            size=Sum("size", default=0)
+        )["size"]
+        return StorageSpace(None, used, None)
 
 
 class StatusView(APIView):

pulpcore/tests/functional/__init__.py

@@ -763,14 +763,15 @@ def __exit__(self, exc_type, exc_value, traceback):
 def random_artifact_factory(
     artifacts_api_client, tmp_path, gen_object_with_cleanup, pulp_domain_enabled
 ):
-    def _random_artifact_factory(pulp_domain=None):
+    def _random_artifact_factory(size=None, pulp_domain=None):
         kwargs = {}
         if pulp_domain:
             if not pulp_domain_enabled:
                 raise RuntimeError("Server does not have domains enabled.")
             kwargs["pulp_domain"] = pulp_domain
         temp_file = tmp_path / str(uuid.uuid4())
-        temp_file.write_bytes(uuid.uuid4().bytes)
+        content = os.urandom(size) if size is not None else uuid.uuid4().bytes
+        temp_file.write_bytes(content)
         return gen_object_with_cleanup(artifacts_api_client, temp_file, **kwargs)
 
     return _random_artifact_factory

pulpcore/tests/functional/api/test_status.py

@@ -33,9 +33,9 @@
         "storage": {
             "type": "object",
             "properties": {
-                "total": {"type": "integer"},
-                "used": {"type": "integer"},
-                "free": {"type": "integer"},
+                "total": {"type": ["integer", "null"]},
+                "used": {"type": ["integer", "null"]},
+                "free": {"type": ["integer", "null"]},
             },
         },
         "content_settings": {
@@ -57,35 +57,25 @@
 }
 
 
-@pytest.fixture(scope="module")
-def expected_pulp_status_schema():
-    """Returns the expected status response."""
-    if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
-        STATUS["properties"]["storage"].pop("properties")
-        STATUS["properties"]["storage"]["type"] = "null"
-
-    return STATUS
-
-
 @pytest.mark.parallel
-def test_get_authenticated(status_api_client, expected_pulp_status_schema):
+def test_get_authenticated(status_api_client):
     """GET the status path with valid credentials.
 
     Verify the response with :meth:`verify_get_response`.
     """
     response = status_api_client.status_read()
-    verify_get_response(response.to_dict(), expected_pulp_status_schema)
+    verify_get_response(response.to_dict(), STATUS)
 
 
 @pytest.mark.parallel
-def test_get_unauthenticated(status_api_client, anonymous_user, expected_pulp_status_schema):
+def test_get_unauthenticated(status_api_client, anonymous_user):
     """GET the status path with no credentials.
 
     Verify the response with :meth:`verify_get_response`.
     """
     with anonymous_user:
         response = status_api_client.status_read()
-    verify_get_response(response.to_dict(), expected_pulp_status_schema)
+    verify_get_response(response.to_dict(), STATUS)
 
 
 @pytest.mark.parallel
@@ -106,6 +96,22 @@ def test_post_authenticated(status_api_client, pulpcore_client, pulp_api_v3_url)
     assert e.value.status == 405
 
 
+@pytest.mark.parallel
+def test_storage_per_domain(status_api_client, domain_factory, random_artifact_factory):
+    """Tests that the storage property returned in status is valid per domain."""
+    domain = domain_factory()
+    domain_status = status_api_client.status_read(pulp_domain=domain.name)
+    assert domain_status["storage"]["used"] == 0
+
+    random_artifact_factory(size=1, pulp_domain=domain.name)
+    domain_status = status_api_client.status_read(pulp_domain=domain.name)
+
+    assert domain_status["storage"]["used"] == 1
+
+    default_status = status_api_client.status_read()
+    assert default_status["storage"] != domain_status["storage"]
+
+
 def verify_get_response(status, expected_schema):
     """Verify the response to an HTTP GET call.
 
@@ -119,3 +125,11 @@ def verify_get_response(status, expected_schema):
     assert status["content_settings"] is not None
     assert status["content_settings"]["content_origin"] is not None
     assert status["content_settings"]["content_path_prefix"] is not None
+
+    assert status["storage"]["used"] is not None
+    if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem":
+        assert status["storage"]["free"] is None
+        assert status["storage"]["total"] is None
+    else:
+        assert status["storage"]["free"] is not None
+        assert status["storage"]["total"] is not None