Activate streaming from object storage

fixes pulp#943

Required PR: ansible/galaxy-importer#168

.github/workflows/ci.yml

@@ -94,8 +94,10 @@ jobs:
           - TEST: docs
           - TEST: azure
           - TEST: s3
+          - TEST: stream
     outputs:
       deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }}
+      deprecations-stream: ${{ steps.deprecations.outputs.deprecations-stream }}
       deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }}
       deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }}
 
@@ -345,13 +347,15 @@ jobs:
       - name: Fail on deprecations
         run: |
           test -z "${{ needs.test.outputs.deprecations-pulp }}"
+          test -z "${{ needs.test.outputs.deprecations-stream }}"
           test -z "${{ needs.test.outputs.deprecations-azure }}"
           test -z "${{ needs.test.outputs.deprecations-s3 }}"
           test -z "${{ needs.test.outputs.deprecations-upgrade }}"
       - name: Print deprecations
         if: failure()
         run: |
           echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d
+          echo "${{ needs.test.outputs.deprecations-stream }}" | base64 -d
           echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d
           echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d
           echo "${{ needs.test.outputs.deprecations-upgrade }}" | base64 -d

.github/workflows/nightly.yml

@@ -26,6 +26,7 @@ jobs:
           - TEST: docs
           - TEST: azure
           - TEST: s3
+          - TEST: stream
           - TEST: generate-bindings
 
     steps:

.github/workflows/release.yml

@@ -84,6 +84,7 @@ jobs:
           - TEST: docs
           - TEST: azure
           - TEST: s3
+          - TEST: stream
           - TEST: generate-bindings
 
     steps:

.github/workflows/scripts/install.sh

@@ -91,6 +91,15 @@ VARSYAML
 if [ "$TEST" = "upgrade" ]; then
   sed -i "/^pulp_container_tag:.*/s//pulp_container_tag: upgrade-https/" vars/main.yaml
 fi
+if [ "$TEST" == 'stream' ]; then
+  sed -i -e '/^services:/a \
+  - name: ci-sftp\
+    image: atmoz/sftp\
+    volumes:\
+      - ./ssh/id_ed25519.pub:/home/foo/.ssh/keys/id_ed25519.pub\
+    command: "foo::::storage"' vars/main.yaml
+  sed -i -e '$a stream_test: true' vars/main.yaml
+fi
 
 if [ "$TEST" = "s3" ]; then
   export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA

CHANGES/943.feature

@@ -0,0 +1 @@
+Enable support for the pulpcore setting ``REDIRECT_TO_OBJECT_STORAGE=False``.

pulp_ansible/app/tasks/collections.py

@@ -1,27 +1,30 @@
 import asyncio
-from collections import defaultdict
-from gettext import gettext as _
 import hashlib
 import json
 import logging
 import tarfile
-import yaml
+from collections import defaultdict
+from gettext import gettext as _
 from operator import attrgetter
 from urllib.parse import urljoin
+from uuid import uuid4
 
+import yaml
 from aiohttp.client_exceptions import ClientResponseError
+from asgiref.sync import sync_to_async
 from async_lru import alru_cache
 from django.conf import settings
 from django.db import transaction
 from django.db.models import Q
+from django.db.utils import IntegrityError
 from django.urls import reverse
 from django.utils.dateparse import parse_datetime
-from galaxy_importer.collection import import_collection as process_collection
 from galaxy_importer.collection import CollectionFilename
+from galaxy_importer.collection import import_collection as process_collection
+from galaxy_importer.collection import sync_collection
 from galaxy_importer.exceptions import ImporterError
+from git import GitCommandError, Repo
 from pkg_resources import Requirement
-from rest_framework.serializers import ValidationError
-
 from pulpcore.plugin.models import (
     Artifact,
     ContentArtifact,
@@ -39,12 +42,13 @@
     DeclarativeArtifact,
     DeclarativeContent,
     DeclarativeVersion,
+    QueryExistingArtifacts,
+    QueryExistingContents,
     RemoteArtifactSaver,
     ResolveContentFutures,
     Stage,
-    QueryExistingArtifacts,
-    QueryExistingContents,
 )
+from rest_framework.serializers import ValidationError
 from semantic_version import Version
 
 from pulp_ansible.app.constants import PAGE_SIZE
@@ -58,22 +62,13 @@
     CollectionVersionSignature,
     Tag,
 )
-from pulp_ansible.app.serializers import CollectionVersionSerializer, CollectionRemoteSerializer
+from pulp_ansible.app.serializers import CollectionRemoteSerializer, CollectionVersionSerializer
 from pulp_ansible.app.tasks.utils import (
+    RequirementsFileEntry,
     get_file_obj_from_tarball,
-    parse_metadata,
     parse_collections_requirements_file,
-    RequirementsFileEntry,
+    parse_metadata,
 )
-from asgiref.sync import sync_to_async
-
-from django.db.utils import IntegrityError
-from galaxy_importer.collection import sync_collection
-from git import GitCommandError, Repo
-
-
-from uuid import uuid4
-
 
 log = logging.getLogger(__name__)
 
@@ -310,6 +305,7 @@ def import_collection(
                     get_file_obj_from_tarball(tar, "FILES.json", temp_file.file.name)
                 )
             url = _get_backend_storage_url(artifact_file)
+            artifact_file.seek(0)
             importer_result = process_collection(
                 artifact_file, filename=filename, file_url=url, logger=user_facing_logger
             )
@@ -397,7 +393,10 @@ def create_collection_from_importer(importer_result, metadata_only=False):
 
 def _get_backend_storage_url(artifact_file):
     """Get artifact url from pulp backend storage."""
-    if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem":
+    if (
+        settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem"
+        or not settings.REDIRECT_TO_OBJECT_STORAGE
+    ):
         url = None
     elif settings.DEFAULT_FILE_STORAGE == "storages.backends.s3boto3.S3Boto3Storage":
         parameters = {"ResponseContentDisposition": "attachment%3Bfilename=archive.tar.gz"}

template_config.yml

@@ -78,7 +78,7 @@ test_performance: false
 test_released_plugin_with_next_pulpcore_release: false
 test_reroute: true
 test_s3: true
-test_stream: false
+test_stream: true
 update_github: true
 update_redmine: false
 upgrade_range: