Running tests on minio

Required PR: pulp/pulp-operator#35
Required PR: pulp/pulpcore#568
https://pulp.plan.io/issues/6155
ref #6155

.travis.yml

@@ -19,6 +19,7 @@ env:
     - TEST=pulp
     - TEST=docs
     - TEST=bindings
+    - TEST=s3
 services:
   - postgresql
   - redis-server

.travis/before_install.sh

@@ -85,12 +85,21 @@ fi
 
 git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master
 
+cd pulpcore
 if [ -n "$PULP_PR_NUMBER" ]; then
-  cd pulpcore
   git fetch --depth=1 origin pull/$PULP_PR_NUMBER/head:$PULP_PR_NUMBER
   git checkout $PULP_PR_NUMBER
-  cd ..
 fi
+if [[ "$TEST" == 's3' ]]; then
+  export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
+  export MINIO_SECRET_KEY=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
+  docker run -d -p 0.0.0.0:9000:9000 -e MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY -e MINIO_SECRET_KEY=$MINIO_SECRET_KEY minio/minio server /data
+  wget https://dl.min.io/client/mc/release/linux-amd64/mc
+  chmod +x mc
+  sudo mv mc /usr/local/bin
+fi
+
+cd ..
 
 
 

.travis/before_script.sh

@@ -31,22 +31,21 @@ else
     sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json
 fi
 
-
-cat ~/.config/pulp_smash/settings.json | \
-    jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json
-cat temp.json > ~/.config/pulp_smash/settings.json
-
-
-if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then
-    # set up pulp-fixtures docker container
-    docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest
-
+if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' ]]; then
     # Many tests require pytest/mock, but users do not need them at runtime
     # (or to add plugins on top of pulpcore or pulp container images.)
     # So install it here, rather than in the image Dockerfile.
     $CMD_PREFIX pip3 install pytest mock
     # Many functional tests require these
     $CMD_PREFIX dnf install -yq lsof which dnf-plugins-core
+
+    # set up pulp-fixtures docker container
+    docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest
+
+    cat ~/.config/pulp_smash/settings.json | \
+        jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json
+    cat temp.json > ~/.config/pulp_smash/settings.json
+
 fi
 
 if [[ -f $POST_BEFORE_SCRIPT ]]; then

.travis/cherrypick.py

@@ -67,7 +67,7 @@ def get_merged_commits(pr):
 g = Github(GITHUB_TOKEN)
 grepo = g.get_repo(REPOSITORY)
 (label,) = (l for l in grepo.get_labels() if l.name == PR_LABEL)
-issues = grepo.get_issues(labels=[label], state="all")
+issues = grepo.get_issues(labels=[label], state="all", sort="updated", direction="asc")
 
 cherrypicks = []
 
@@ -91,9 +91,9 @@ def get_merged_commits(pr):
         if ret.returncode != 0:
             print(f"Failed to cherry-pick commit {commit.sha}: {ret.stderr.decode('ascii')}")
             exit(1)
-        else:
-            cherrypicks.append(issue)
-            print(f"Cherry-picked commit {commit.sha}.")
+
+    cherrypicks.append(issue)
+    print(f"Cherry-picked commit {commit.sha}.")
 
 # check if we cherry picked anything
 if len(cherrypicks) == 0:

.travis/install.sh

@@ -54,6 +54,7 @@ images:
   - pulp_file-${TAG}:
       image_name: pulp_file
       tag: $TAG
+      pulpcore: pulpcore
       plugins:
         - ./pulp_file
 VARSYAML
@@ -69,6 +70,11 @@ images:
         - ./pulp_file
 VARSYAML
 fi
+
+if [ "$TEST" = 's3' ]; then
+  echo "s3_test: true" >> vars/vars.yaml
+fi
+
 ansible-playbook -v build.yaml
 
 cd $TRAVIS_BUILD_DIR/../pulp-operator
@@ -93,6 +99,40 @@ spec:
     admin_password: pulp
 CRYAML
 
+if [ "$TEST" = 's3' ]; then
+  cat > deploy/crds/pulpproject_v1alpha1_pulp_cr.yaml << CRYAML
+  apiVersion: pulpproject.org/v1alpha1
+  kind: Pulp
+  metadata:
+    name: example-pulp
+  spec:
+    pulp_file_storage:
+      # k3s local-path requires this
+      access_mode: "ReadWriteOnce"
+      # We have a little over 40GB free on Travis VMs/instances
+      size: "40Gi"
+    image: pulp_file
+    tag: "${TAG}"
+    database_connection:
+      username: pulp
+      password: pulp
+      admin_password: pulp
+    pulp_settings:
+      aws_access_key_id: "AKIAIT2Z5TDYPX3ARJBA"
+      aws_secret_access_key: "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"
+      aws_storage_bucket_name: "pulp3"
+      aws_default_acl: "@none None"
+      s3_use_sigv4: true
+      aws_s3_signature_version: "s3v4"
+      aws_s3_addressing_style: "path"
+      aws_s3_region_name: "eu-central-1"
+      default_file_storage: "storages.backends.s3boto3.S3Boto3Storage"
+      media_root: ''
+      aws_s3_endpoint_url: "http://$(hostname):9000"
+
+CRYAML
+fi
+
 # Install k3s, lightweight Kubernetes
 .travis/k3s-install.sh
 # Deploy pulp-operator, with the pulp containers, according to CRYAML

.travis/publish_client_gem.sh

@@ -23,7 +23,10 @@ if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then
   export VERSION=${REPORTED_VERSION}
 else
   # Daily publishing of development version (ends in ".dev" reported as ".dev0")
-  [ "${REPORTED_VERSION%.dev*}" != "${REPORTED_VERSION}" ] || exit 1
+  if [ "${REPORTED_VERSION%.dev*}" == "${REPORTED_VERSION}" ]; then
+    echo "Refusing to publish bindings. $REPORTED_VERSION does not contain 'dev'."
+    exit 1
+  fi
   export EPOCH="$(date +%s)"
   export VERSION=${REPORTED_VERSION}${EPOCH}
 fi

.travis/publish_client_pypi.sh

@@ -21,7 +21,10 @@ if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then
   export VERSION=${REPORTED_VERSION}
 else
   # Daily publishing of development version (ends in ".dev" reported as ".dev0")
-  [ "${REPORTED_VERSION%.dev*}" != "${REPORTED_VERSION}" ] || exit 1
+  if [ "${REPORTED_VERSION%.dev*}" == "${REPORTED_VERSION}" ]; then
+    echo "Refusing to publish bindings. $REPORTED_VERSION does not contain 'dev'."
+    exit 1
+  fi
   export EPOCH="$(date +%s)"
   export VERSION=${REPORTED_VERSION}${EPOCH}
 fi

.travis/script.sh

@@ -89,6 +89,13 @@ export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --"
 cat unittest_requirements.txt | $CMD_STDIN_PREFIX bash -c "cat > /tmp/test_requirements.txt"
 $CMD_PREFIX pip3 install -r /tmp/test_requirements.txt
 
+if [[ "$TEST" == 's3' ]]; then
+  mc config host add s3 http://localhost:9000 AKIAIT2Z5TDYPX3ARJBA fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS --api S3v4
+  mc config host rm local
+  mc mb s3/pulp3 --region eu-central-1
+  mc tree s3
+fi
+
 # Run unit tests.
 $CMD_PREFIX bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/pulp_file/tests/unit/"
 
@@ -107,11 +114,12 @@ export PYTHONPATH=$TRAVIS_BUILD_DIR:$TRAVIS_BUILD_DIR/../pulpcore:${PYTHONPATH}
 set -u
 
 if [[ "$TEST" == "performance" ]]; then
+  wget -qO- https://github.com/crazy-max/travis-wait-enhanced/releases/download/v1.0.0/travis-wait-enhanced_1.0.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -zxvf - travis-wait-enhanced
   echo "--- Performance Tests ---"
   if [[ -z ${PERFORMANCE_TEST+x} ]]; then
-    pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_file.tests.performance || show_logs_and_return_non_zero
+    travis-wait-enhanced --interval=1m --timeout=30m -- pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_file.tests.performance || show_logs_and_return_non_zero
   else
-    pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_file.tests.performance.test_$PERFORMANCE_TEST || show_logs_and_return_non_zero
+    travis-wait-enhanced --interval=1m --timeout=30m -- pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_file.tests.performance.test_$PERFORMANCE_TEST || show_logs_and_return_non_zero
   fi
   exit
 fi

CHANGES/6155.misc

@@ -0,0 +1 @@
+Running tests on minio

pulp_file/tests/functional/utils.py

@@ -173,7 +173,7 @@ def monitor_task(task_href):
     completed = ["completed", "failed", "canceled"]
     task = tasks.read(task_href)
     while task.state not in completed:
-        sleep(2)
+        sleep(1)
         task = tasks.read(task_href)
 
     if task.state == "completed":

pulp_file/tests/unit/test_core/test_repo_version.py

@@ -1,5 +1,6 @@
 from unittest.mock import patch
 
+from django.core.files.storage import default_storage as storage
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase
 from pulpcore.plugin.models import Artifact, ContentArtifact, RepositoryVersion
@@ -36,8 +37,9 @@ def setUp(self):
         artifact.save()
         self.content = FileContent.objects.create()
         self.content.save()
+        artifact_file = storage.open(artifact.file.name)
         self.content_artifact = ContentArtifact.objects.create(
-            artifact=artifact, content=self.content, relative_path=artifact.file.path
+            artifact=artifact, content=self.content, relative_path=artifact_file.name
         )
         self.content_artifact.save()
         self.repository = FileRepository.objects.create(name="foo")

template_config.yml

@@ -28,12 +28,15 @@ plugin_snake: pulp_file
 plugin_snake_short: file
 pulp_settings: null
 pulpcore_branch: master
+pulpcore_pip_version_specifier: null
 pydocstyle: true
 pypi_username: pulp
 stable_branch: '0.2'
 test: false
 test_bindings: true
 test_performance: true
+test_s3: true
+travis_addtl_services: []
 travis_notifications:
   irc:
     channels: