Introducing tests against minio

Required PR: pulp/pulpcore#568
https://pulp.plan.io/issues/5962
closes #5962

.travis.yml

@@ -17,6 +17,7 @@ python:
 env:
   matrix:
     - TEST=pulp
+    - TEST=s3
 services:
   - postgresql
   - redis-server
@@ -26,7 +27,6 @@ addons:
     packages:
       - httpie
       - jq
-      - nginx
   # postgres versions provided by el7 RHSCL (lowest supportable version)
   postgresql: "9.6"
 before_install: .travis/before_install.sh

.travis/before_install.sh

@@ -85,13 +85,23 @@ fi
 
 git clone --depth=1 https://github.com/pulp/pulpcore.git
 
+cd pulpcore
 if [ -n "$PULP_PR_NUMBER" ]; then
-  cd pulpcore
   git fetch --depth=1 origin pull/$PULP_PR_NUMBER/head:$PULP_PR_NUMBER
   git checkout $PULP_PR_NUMBER
-  cd ..
 fi
 
+if [[ "$TEST" == 's3' ]]; then
+  export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
+  export MINIO_SECRET_KEY=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
+  docker run -d -p 0.0.0.0:9000:9000 -e MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY -e MINIO_SECRET_KEY=$MINIO_SECRET_KEY minio/minio server /data
+  wget https://dl.min.io/client/mc/release/linux-amd64/mc
+  chmod +x mc
+  sudo mv mc /usr/local/bin
+fi
+
+cd ..
+
 rm ./pulpcore/containers/images/pulp/container-assets/pulp-api
 mv ./plugin_template/.travis/pulp-api ./pulpcore/containers/images/pulp/container-assets/pulp-api
 

.travis/pre_before_install.sh

@@ -5,6 +5,7 @@ pip install -r test_requirements.txt
 mkdir ../pulp_catdog/.travis
 touch ../pulp_catdog/.travis/test_bindings.py
 echo 'pypi_username: the_pypi_user' >> ../pulp_catdog/template_config.yml
+sed -i "s/test_s3: false/test_s3: true/g" ../pulp_catdog/template_config.yml
 ./plugin-template --all pulp_catdog
 mv ../pulp_catdog/*.txt .
 mv ../pulp_catdog/.travis/before_install.sh ../pulp_catdog/.travis/old_before_install.sh

README.md

@@ -91,6 +91,9 @@ The following settings are stored in `template_config.yml`.
                         script in the .travis directory of the plugin repository. This script
                         is supposed to exercise the generated client library.
 
+  test_s3               Include s3 job for running tests using [minio](https://github.com/minio/minio)
+                        to emulate S3.
+
   test_performance      Include a Travis stage that runs a script to test performance. If using a
                         list, a separate stage will run a specific performance test file for each
                         entry in the list. Otherwise, all performance tests will be run together.

plugin-template

@@ -29,6 +29,7 @@ DEFAULT_SETTINGS = {
     'docs_test': True,
     'pydocstyle': True,
     'test_bindings': True,
+    'test_s3': False,
     'test_performance': False,
     'plugin_camel': None,
     'plugin_camel_short': None,
@@ -44,10 +45,24 @@ DEFAULT_SETTINGS = {
     'pulpcore_pip_version_specifier': None,
     'pypi_username': None,
     'stable_branch': None,
+    's3_settings': {
+        'aws_access_key_id': 'AKIAIT2Z5TDYPX3ARJBA',
+        'aws_secret_access_key': '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',
+        'aws_storage_bucket_name': 'pulp3',
+        'aws_default_acl': '"@none None"',
+        's3_use_sigv4': True,
+        'aws_s3_signature_version': 's3v4',
+        'aws_s3_addressing_style': 'path',
+        'aws_s3_region_name': 'eu-central-1',
+        'default_file_storage': 'storages.backends.s3boto3.S3Boto3Storage',
+        'media_root': '',
+        'aws_s3_endpoint_url': 'http://$(hostname):9000',
+    },
     'travis_addtl_services': [],
     'travis_notifications': None,
 }
 
+
 def main():
     parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
                                      description='Create or update a plugin using the current '

templates/travis/.travis.yml.j2

@@ -18,6 +18,9 @@ env:
     {%- if test_bindings %}
     - TEST=bindings
     {%- endif %}
+    {%- if test_s3 %}
+    - TEST=s3
+    {%- endif %}
 {%- if test_bindings or docs_test %}
 {%- endif %}
 services:

templates/travis/.travis/before_install.sh.j2

@@ -85,12 +85,22 @@ fi
 {% if plugin_name != 'pulpcore' -%}
 git clone --depth=1 https://github.com/pulp/pulpcore.git --branch {{ pulpcore_branch }}
 
+cd pulpcore
 if [ -n "$PULP_PR_NUMBER" ]; then
-  cd pulpcore
   git fetch --depth=1 origin pull/$PULP_PR_NUMBER/head:$PULP_PR_NUMBER
   git checkout $PULP_PR_NUMBER
-  cd ..
 fi
+{% if test_s3 -%}
+if [[ "$TEST" == 's3' ]]; then
+  export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
+  export MINIO_SECRET_KEY=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
+  docker run -d -p 0.0.0.0:9000:9000 -e MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY -e MINIO_SECRET_KEY=$MINIO_SECRET_KEY minio/minio server /data
+  wget https://dl.min.io/client/mc/release/linux-amd64/mc
+  chmod +x mc
+  sudo mv mc /usr/local/bin
+fi
+{% endif %}
+cd ..
 {% endif %}
 
 {% if plugin_name != 'pulpcore' %}

templates/travis/.travis/before_script.sh.j2

@@ -26,23 +26,21 @@ else
     sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json
 fi
 
-{% if docker_fixtures %}
-# set up pulp-fixtures docker container
-if [[ "$TEST" == 'pulp' ]]; then
-  docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest
-fi
-cat ~/.config/pulp_smash/settings.json | \
-    jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json
-cat temp.json > ~/.config/pulp_smash/settings.json
-{% endif %}
-
-if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then
+if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' ]]; then
     # Many tests require pytest/mock, but users do not need them at runtime
     # (or to add plugins on top of pulpcore or pulp container images.)
     # So install it here, rather than in the image Dockerfile.
     $CMD_PREFIX pip3 install pytest mock
     # Many functional tests require these
     $CMD_PREFIX dnf install -yq lsof which dnf-plugins-core
+    {% if docker_fixtures %}
+    # set up pulp-fixtures docker container
+    docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest
+
+    cat ~/.config/pulp_smash/settings.json | \
+        jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json
+    cat temp.json > ~/.config/pulp_smash/settings.json
+    {% endif %}
 fi
 
 if [[ -f $POST_BEFORE_SCRIPT ]]; then

templates/travis/.travis/install.sh.j2

@@ -101,6 +101,11 @@ images:
         {%- endfor %}
 VARSYAML
 {%- endif %}
+
+if [ "$TEST" = 's3' ]; then
+  echo "s3_test: true" >> vars/vars.yaml
+fi
+
 ansible-playbook -v build.yaml
 
 cd $TRAVIS_BUILD_DIR/../pulp-operator
@@ -130,6 +135,38 @@ spec:
 {%- endif %}
 CRYAML
 
+if [ "$TEST" = 's3' ]; then
+  cat > deploy/crds/pulpproject_v1alpha1_pulp_cr.yaml << CRYAML
+  apiVersion: pulpproject.org/v1alpha1
+  kind: Pulp
+  metadata:
+    name: example-pulp
+  spec:
+    pulp_file_storage:
+      # k3s local-path requires this
+      access_mode: "ReadWriteOnce"
+      # We have a little over 40GB free on Travis VMs/instances
+      size: "40Gi"
+    image: {% if plugin_name != 'pulpcore' %}{{ plugin_name }}{% else %}pulp_file{% endif %}
+    tag: "${TAG}"
+    database_connection:
+      username: pulp
+      password: pulp
+      admin_password: pulp
+  {%- if pulp_settings or test_s3 %}
+    pulp_settings:
+      {% if pulp_settings %}
+      {%- for setting, value in pulp_settings.items() -%} {{ setting }}: {{ value }}
+      {% endfor -%}
+      {% endif -%}
+      {% if test_s3 %}
+      {%- for setting, value in s3_settings.items() -%} {{ setting }}: {{ value if value else "''" }}
+      {% endfor -%}
+      {% endif -%}
+  {%- endif %}
+CRYAML
+fi
+
 # Install k3s, lightweight Kubernetes
 .travis/k3s-install.sh
 # Deploy pulp-operator, with the pulp containers, according to CRYAML

templates/travis/.travis/script.sh.j2

@@ -94,6 +94,13 @@ export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --"
 cat unittest_requirements.txt | $CMD_STDIN_PREFIX bash -c "cat > /tmp/test_requirements.txt"
 $CMD_PREFIX pip3 install -r /tmp/test_requirements.txt
 
+if [[ "$TEST" == 's3' ]]; then
+  mc config host add s3 http://localhost:9000 AKIAIT2Z5TDYPX3ARJBA fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS --api S3v4
+  mc config host rm local
+  mc mb s3/pulp3 --region eu-central-1
+  mc tree s3
+fi
+
 # Run unit tests.
 $CMD_PREFIX bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/{{ plugin_snake }}/tests/unit/"