From fc30edadfffe85fba26a7963a30ca18ffef031a1 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 12 Sep 2023 14:34:28 -0400 Subject: [PATCH] Update storage in status to be per domain for all storage types fixes: #4456 fixes: #4457 --- CHANGES/4456.feature | 1 + CHANGES/4457.feature | 1 + pulpcore/app/serializers/status.py | 12 +++-- pulpcore/app/views/status.py | 19 ++++++-- pulpcore/tests/functional/__init__.py | 5 +- pulpcore/tests/functional/api/test_status.py | 48 +++++++++++++------- 6 files changed, 59 insertions(+), 27 deletions(-) create mode 100644 CHANGES/4456.feature create mode 100644 CHANGES/4457.feature diff --git a/CHANGES/4456.feature b/CHANGES/4456.feature new file mode 100644 index 0000000000..791916c8d3 --- /dev/null +++ b/CHANGES/4456.feature @@ -0,0 +1 @@ +Status endpoint now reports storage usage of the domain being called from. \ No newline at end of file diff --git a/CHANGES/4457.feature b/CHANGES/4457.feature new file mode 100644 index 0000000000..47475bf8ed --- /dev/null +++ b/CHANGES/4457.feature @@ -0,0 +1 @@ +Status endpoint now reports the storage usage for non-filesystem storage backends. \ No newline at end of file diff --git a/pulpcore/app/serializers/status.py b/pulpcore/app/serializers/status.py index 4a6e072679..cc1789074d 100644 --- a/pulpcore/app/serializers/status.py +++ b/pulpcore/app/serializers/status.py @@ -50,11 +50,17 @@ class StorageSerializer(serializers.Serializer): Serializer for information about the storage system """ - total = serializers.IntegerField(min_value=0, help_text=_("Total number of bytes")) + total = serializers.IntegerField( + min_value=0, help_text=_("Total number of bytes"), allow_null=True + ) - used = serializers.IntegerField(min_value=0, help_text=_("Number of bytes in use")) + used = serializers.IntegerField( + min_value=0, help_text=_("Number of bytes in use"), allow_null=True + ) - free = serializers.IntegerField(min_value=0, help_text=_("Number of free bytes")) + free = serializers.IntegerField( + min_value=0, help_text=_("Number of free bytes"), allow_null=True + ) class ContentSettingsSerializer(serializers.Serializer): diff --git a/pulpcore/app/views/status.py b/pulpcore/app/views/status.py index 11902b5a49..a021e5487d 100644 --- a/pulpcore/app/views/status.py +++ b/pulpcore/app/views/status.py @@ -3,28 +3,37 @@ from gettext import gettext as _ from django.conf import settings -from django.core.files.storage import default_storage +from django.db.models import Sum from drf_spectacular.utils import extend_schema from rest_framework.response import Response from rest_framework.views import APIView +from collections import namedtuple from pulpcore.app.apps import pulp_plugin_configs +from pulpcore.app.models.content import Artifact from pulpcore.app.models.status import ApiAppStatus, ContentAppStatus from pulpcore.app.models.task import Worker from pulpcore.app.serializers.status import StatusSerializer from pulpcore.app.redis_connection import get_redis_connection +from pulpcore.app.util import get_domain _logger = logging.getLogger(__name__) +StorageSpace = namedtuple("StorageSpace", ("total", "used", "free")) def _disk_usage(): - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + domain = get_domain() + if domain.storage_class == "pulpcore.app.models.storage.FileSystem": + storage = domain.get_storage() try: - return shutil.disk_usage(default_storage.location) + return shutil.disk_usage(storage.location) except Exception: _logger.exception(_("Failed to determine disk usage")) - - return None + else: + used = Artifact.objects.filter(pulp_domain=domain).aggregate( + size=Sum("size", default=0) + )["size"] + return StorageSpace(None, used, None) class StatusView(APIView): diff --git a/pulpcore/tests/functional/__init__.py b/pulpcore/tests/functional/__init__.py index 810e6715c4..159cad95bd 100644 --- a/pulpcore/tests/functional/__init__.py +++ b/pulpcore/tests/functional/__init__.py @@ -763,14 +763,15 @@ def __exit__(self, exc_type, exc_value, traceback): def random_artifact_factory( artifacts_api_client, tmp_path, gen_object_with_cleanup, pulp_domain_enabled ): - def _random_artifact_factory(pulp_domain=None): + def _random_artifact_factory(size=None, pulp_domain=None): kwargs = {} if pulp_domain: if not pulp_domain_enabled: raise RuntimeError("Server does not have domains enabled.") kwargs["pulp_domain"] = pulp_domain temp_file = tmp_path / str(uuid.uuid4()) - temp_file.write_bytes(uuid.uuid4().bytes) + content = os.urandom(size) if size is not None else uuid.uuid4().bytes + temp_file.write_bytes(content) return gen_object_with_cleanup(artifacts_api_client, temp_file, **kwargs) return _random_artifact_factory diff --git a/pulpcore/tests/functional/api/test_status.py b/pulpcore/tests/functional/api/test_status.py index 4e37189ead..6df096c932 100644 --- a/pulpcore/tests/functional/api/test_status.py +++ b/pulpcore/tests/functional/api/test_status.py @@ -33,9 +33,9 @@ "storage": { "type": "object", "properties": { - "total": {"type": "integer"}, - "used": {"type": "integer"}, - "free": {"type": "integer"}, + "total": {"type": ["integer", "null"]}, + "used": {"type": ["integer", "null"]}, + "free": {"type": ["integer", "null"]}, }, }, "content_settings": { @@ -57,35 +57,25 @@ } -@pytest.fixture(scope="module") -def expected_pulp_status_schema(): - """Returns the expected status response.""" - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": - STATUS["properties"]["storage"].pop("properties") - STATUS["properties"]["storage"]["type"] = "null" - - return STATUS - - @pytest.mark.parallel -def test_get_authenticated(status_api_client, expected_pulp_status_schema): +def test_get_authenticated(status_api_client): """GET the status path with valid credentials. Verify the response with :meth:`verify_get_response`. """ response = status_api_client.status_read() - verify_get_response(response.to_dict(), expected_pulp_status_schema) + verify_get_response(response.to_dict(), STATUS) @pytest.mark.parallel -def test_get_unauthenticated(status_api_client, anonymous_user, expected_pulp_status_schema): +def test_get_unauthenticated(status_api_client, anonymous_user): """GET the status path with no credentials. Verify the response with :meth:`verify_get_response`. """ with anonymous_user: response = status_api_client.status_read() - verify_get_response(response.to_dict(), expected_pulp_status_schema) + verify_get_response(response.to_dict(), STATUS) @pytest.mark.parallel @@ -106,6 +96,22 @@ def test_post_authenticated(status_api_client, pulpcore_client, pulp_api_v3_url) assert e.value.status == 405 +@pytest.mark.parallel +def test_storage_per_domain(status_api_client, domain_factory, random_artifact_factory): + """Tests that the storage property returned in status is valid per domain.""" + domain = domain_factory() + domain_status = status_api_client.status_read(pulp_domain=domain.name) + assert domain_status["storage"]["used"] == 0 + + random_artifact_factory(size=1, pulp_domain=domain.name) + domain_status = status_api_client.status_read(pulp_domain=domain.name) + + assert domain_status["storage"]["used"] == 1 + + default_status = status_api_client.status_read() + assert default_status["storage"] != domain_status["storage"] + + def verify_get_response(status, expected_schema): """Verify the response to an HTTP GET call. @@ -119,3 +125,11 @@ def verify_get_response(status, expected_schema): assert status["content_settings"] is not None assert status["content_settings"]["content_origin"] is not None assert status["content_settings"]["content_path_prefix"] is not None + + assert status["storage"]["used"] is not None + if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + assert status["storage"]["free"] is None + assert status["storage"]["total"] is None + else: + assert status["storage"]["free"] is not None + assert status["storage"]["total"] is not None