Running tests on minio

Required PR: pulp/pulp_file#358
https://pulp.plan.io/issues/6155
closes #6155

.travis.yml

@@ -19,6 +19,7 @@ env:
     - TEST=pulp
     - TEST=docs
     - TEST=bindings
+    - TEST=s3
 services:
   - postgresql
   - redis-server

.travis/before_script.sh

@@ -31,23 +31,21 @@ else
     sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json
 fi
 
-
-# set up pulp-fixtures docker container
-if [[ "$TEST" == 'pulp' ]]; then
-  docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest
-fi
-cat ~/.config/pulp_smash/settings.json | \
-    jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json
-cat temp.json > ~/.config/pulp_smash/settings.json
-
-
-if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then
+if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' ]]; then
     # Many tests require pytest/mock, but users do not need them at runtime
     # (or to add plugins on top of pulpcore or pulp container images.)
     # So install it here, rather than in the image Dockerfile.
     $CMD_PREFIX pip3 install pytest mock
     # Many functional tests require these
     $CMD_PREFIX dnf install -yq lsof which dnf-plugins-core
+
+    # set up pulp-fixtures docker container
+    docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest
+
+    cat ~/.config/pulp_smash/settings.json | \
+        jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json
+    cat temp.json > ~/.config/pulp_smash/settings.json
+
 fi
 
 if [[ -f $POST_BEFORE_SCRIPT ]]; then

.travis/install.sh

@@ -69,6 +69,11 @@ images:
         - $PULP_FILE
         - $PULP_CERTGUARD
 VARSYAML
+
+if [ "$TEST" = 's3' ]; then
+  echo "s3_test: true" > vars/vars.yaml
+fi
+
 ansible-playbook -v build.yaml
 
 cd $TRAVIS_BUILD_DIR/../pulp-operator

.travis/pre_before_install.sh

@@ -26,3 +26,13 @@ if [ -n "$PULP_CERTGUARD_PR_NUMBER" ]; then
 fi
 
 cd pulpcore
+
+if [[ "$TEST" == 's3' ]]; then
+  sed -i "s/localhost/$(hostname)/g" ./.travis/s3_settings.py
+  export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
+  export MINIO_SECRET_KEY=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
+  docker run -d -p 0.0.0.0:9000:9000 -e MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY -e MINIO_SECRET_KEY=$MINIO_SECRET_KEY minio/minio server /data
+  wget https://dl.min.io/client/mc/release/linux-amd64/mc
+  chmod +x mc
+  sudo mv mc /usr/local/bin
+fi

.travis/s3_settings.py

@@ -0,0 +1,20 @@
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'pulp',
+        'USER': 'pulp',
+        'CONN_MAX_AGE': 0,
+    },
+}
+CONTENT_ORIGIN = "http://localhost:24816"
+AWS_ACCESS_KEY_ID = 'AKIAIT2Z5TDYPX3ARJBA'
+AWS_SECRET_ACCESS_KEY = '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'
+AWS_STORAGE_BUCKET_NAME = 'pulp3'
+AWS_DEFAULT_ACL = None
+S3_USE_SIGV4 = True
+AWS_S3_SIGNATURE_VERSION = 's3v4'
+AWS_S3_ADDRESSING_STYLE = 'path'
+AWS_S3_REGION_NAME = 'eu-central-1'
+DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
+MEDIA_ROOT = ''
+AWS_S3_ENDPOINT_URL = 'http://localhost:9000'

.travis/script.sh

@@ -89,6 +89,13 @@ export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --"
 cat unittest_requirements.txt | $CMD_STDIN_PREFIX bash -c "cat > /tmp/test_requirements.txt"
 $CMD_PREFIX pip3 install -r /tmp/test_requirements.txt
 
+if [[ "$TEST" == 's3' ]]; then
+  mc config host add s3 http://localhost:9000 AKIAIT2Z5TDYPX3ARJBA fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS --api S3v4
+  mc config host rm local
+  mc mb s3/pulp3 --region eu-central-1
+  mc tree s3
+fi
+
 # Run unit tests.
 $CMD_PREFIX bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/pulpcore/tests/unit/"
 

containers/images/pulp/Dockerfile.j2

@@ -48,6 +48,11 @@ RUN pip install --upgrade pip
 RUN mkdir -p /etc/pulp
 
 RUN pip install gunicorn
+{% if s3_test is defined %}
+RUN pip install django-storages[boto3]
+# Hacking botocore (https://github.com/boto/botocore/pull/1990):
+RUN sed -i "s/hasattr(body, 'read')/getattr(body, '_size', None)/g" $(pip show botocore | grep -i location | awk '{ print $2 }')/botocore/handlers.py
+{% endif %}
 
 {% if ( item.value.pulpcore is defined and
         item.value.pulpcore.startswith('./') ) %}
@@ -70,6 +75,9 @@ RUN pip install {{ plugin }}
 {% endfor %}
 {% endif %}
 
+{% if s3_test is defined %}
+COPY pulpcore/.travis/s3_settings.py /etc/pulp/settings.py
+{% endif %}
 COPY pulpcore/containers/images/pulp/container-assets/wait_on_postgres.py /usr/bin/wait_on_postgres.py
 COPY pulpcore/containers/images/pulp/container-assets/wait_on_database_migrations.sh /usr/bin/wait_on_database_migrations.sh
 COPY pulpcore/containers/images/pulp/container-assets/pulp-common-entrypoint.sh /pulp-common-entrypoint.sh

pulpcore/tests/unit/models/test_content.py

@@ -1,6 +1,7 @@
 import os
 import tempfile
 
+from django.core.files.storage import default_storage as storage
 from django.test import TestCase
 from pulpcore.plugin.models import Artifact, Content, ContentArtifact
 
@@ -23,10 +24,11 @@ def setUp(self):
     def test_create_and_read_content(self):
         content = Content.objects.create()
         content.save()
+        artifact_file = storage.open(self.artifact01.file.name)
         content_artifact = ContentArtifact.objects.create(
             artifact=self.artifact01,
             content=content,
-            relative_path=self.artifact01.file.path)
+            relative_path=artifact_file.name)
         content_artifact.save()
         self.assertTrue(
             Content.objects.filter(pk=content.pk).exists()

template_config.yml

@@ -1,14 +1,11 @@
 # This config represents the latest values used when running the plugin-template. Any settings that
 # were not present before running plugin-template have been added with their default values.
 
-# pulp_file is hardcoded in plugin_template when generating for pulpcore in some
-# places. In other places, plugin_template uses generic logic for it, so we
-# specify it here.
-# Also, .travis/pre_before_install.sh reads the pulp_file branch. At runtime,
-# not template generation time.
 additional_plugins:
-- { name: pulp_file, branch: master }
-- { name: pulp-certguard, branch: master }
+- branch: master
+  name: pulp_file
+- branch: master
+  name: pulp-certguard
 black: false
 check_commit_message: true
 cherry_pick_automation: true
@@ -31,11 +28,14 @@ plugin_name: pulpcore
 plugin_snake: pulpcore
 pulp_settings: null
 pulpcore_branch: null
+pulpcore_pip_version_specifier: null
 pydocstyle: true
 pypi_username: pulp
-stable_branch: "3.2"
+stable_branch: '3.2'
 test_bindings: true
 test_performance: false
+test_s3: true
+travis_addtl_services: []
 travis_notifications:
   irc:
     channels: