Merge pull request #237 from fabricio-aguiar/5579

Add the ability to set a certification status

.travis.yml

@@ -15,16 +15,15 @@ python:
     - "3.7"
 env:
   matrix:
-    - DB=postgres TEST=pulp
-    - DB=postgres TEST=docs
-    - DB=postgres TEST=bindings
-
+    - TEST=pulp
+    - TEST=docs
+    - TEST=bindings
 matrix:
   exclude:
     - python: '3.7'
-      env: DB=postgres TEST=bindings
+      env: TEST=bindings
     - python: '3.6'
-      env: DB=postgres TEST=docs
+      env: TEST=docs
   fast_finish: true
 services:
     - postgresql
@@ -53,25 +52,21 @@ jobs:
   - stage: publish-daily-client-gem
     script: PULP_SETTINGS=/etc/pulp/settings.py bash .travis/publish_client_gem.sh
     env:
-      - DB=postgres
       - TEST=bindings
     if: type = cron
   - stage: publish-daily-client-pypi
     script: PULP_SETTINGS=/etc/pulp/settings.py bash .travis/publish_client_pypi.sh
     env:
-      - DB=postgres
       - TEST=bindings
     if: type = cron
   - stage: publish-client-gem
     script: PULP_SETTINGS=/etc/pulp/settings.py bash .travis/publish_client_gem.sh
     env:
-      - DB=postgres
       - TEST=bindings
     if: tag IS present
   - stage: publish-client-pypi
     script: PULP_SETTINGS=/etc/pulp/settings.py bash .travis/publish_client_pypi.sh
     env:
-      - DB=postgres
       - TEST=bindings
     if: tag IS present
 

.travis/before_install.sh

@@ -19,13 +19,28 @@ if [ -f $PRE_BEFORE_INSTALL ]; then
     $PRE_BEFORE_INSTALL
 fi
 
-export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
-export PULP_PLUGIN_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore-plugin\/pull\/(\d+)' | awk -F'/' '{print $7}')
-export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
-export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}')
-export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
+if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*" | grep -v "https") ]]; then
+  echo "Invalid Required PR link detected in commit message. Please use the full https url."
+  exit 1
+fi
+
+if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ -z "$TRAVIS_TAG" -a "$TRAVIS_BRANCH" != "master"]
+then
+  export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/PulpQE\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  export PULP_OPERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-operator\/pull\/(\d+)' | awk -F'/' '{print $7}')
+else
+  export PULP_PR_NUMBER=
+  export PULP_SMASH_PR_NUMBER=
+  export PULP_ROLES_PR_NUMBER=
+  export PULP_BINDINGS_PR_NUMBER=
+  export PULP_OPERATOR_PR_NUMBER=
+fi
 
-# dev_requirements should not be needed for testing; don't install them to make sure
+# test_requirements contains tools needed for flake8, etc.
+# So install them here rather than in install.sh
 pip install -r test_requirements.txt
 
 # check the commit message
@@ -39,48 +54,49 @@ flake8 --config flake8.cfg
 
 cd ..
 git clone --depth=1 https://github.com/pulp/ansible-pulp.git
-if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
-  if [ -n "$PULP_ROLES_PR_NUMBER" ]; then
-    cd ansible-pulp
-    git fetch --depth=1 origin +refs/pull/$PULP_ROLES_PR_NUMBER/merge
-    git checkout FETCH_HEAD
-    cd ..
-  fi
+if [ -n "$PULP_ROLES_PR_NUMBER" ]; then
+  cd ansible-pulp
+  git fetch --depth=1 origin +refs/pull/$PULP_ROLES_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  cd ..
 fi
 
-git clone --depth=1 https://github.com/pulp/pulpcore.git
 
-if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
-  if [ -n "$PULP_PR_NUMBER" ]; then
-    cd pulpcore
-    git fetch --depth=1 origin +refs/pull/$PULP_PR_NUMBER/merge
-    git checkout FETCH_HEAD
-    cd ..
-  fi
+git clone --depth=1 https://github.com/pulp/pulp-operator.git
+if [ -n "$PULP_OPERATOR_PR_NUMBER" ]; then
+  cd pulp-operator
+  git fetch --depth=1 origin +refs/pull/$PULP_OPERATOR_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  RELEASE_VERSION=v0.9.0
+  curl -LO https://github.com/operator-framework/operator-sdk/releases/download/${RELEASE_VERSION}/operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
+  chmod +x operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu && sudo mkdir -p /usr/local/bin/ && sudo cp operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu /usr/local/bin/operator-sdk && rm operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
+  sudo operator-sdk build --image-builder=docker quay.io/pulp/pulp-operator:latest
+  cd ..
 fi
 
 
-git clone --depth=1 https://github.com/pulp/pulpcore-plugin.git
+git clone --depth=1 https://github.com/pulp/pulpcore.git
 
-if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
-  if [ -n "$PULP_PLUGIN_PR_NUMBER" ]; then
-    cd pulpcore-plugin
-    git fetch --depth=1 origin +refs/pull/$PULP_PLUGIN_PR_NUMBER/merge
-    git checkout FETCH_HEAD
-    cd ..
-  fi
+if [ -n "$PULP_PR_NUMBER" ]; then
+  cd pulpcore
+  git fetch --depth=1 origin +refs/pull/$PULP_PR_NUMBER/merge
+  git checkout FETCH_HEAD
+  cd ..
 fi
 
 
-git clone --depth=1 https://github.com/PulpQE/pulp-smash.git
+# When building a (release) tag, we don't need the development modules for the
+# build (they will be installed as dependencies of the plugin).
+if [ -z "$TRAVIS_TAG" ]; then
+  git clone --depth=1 https://github.com/PulpQE/pulp-smash.git
 
-if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
   if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
     cd pulp-smash
     git fetch --depth=1 origin +refs/pull/$PULP_SMASH_PR_NUMBER/merge
     git checkout FETCH_HEAD
     cd ..
   fi
+
   # pulp-smash already got installed via test_requirements.txt
   pip install --upgrade --force-reinstall ./pulp-smash
 fi
@@ -89,7 +105,6 @@ psql -c 'CREATE DATABASE pulp OWNER travis;'
 
 pip install ansible
 cp pulp_ansible/.travis/playbook.yml ansible-pulp/playbook.yml
-cp pulp_ansible/.travis/postgres.yml ansible-pulp/postgres.yml
 
 cd pulp_ansible
 

.travis/check_commit.sh

@@ -7,6 +7,8 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+set -euv
+
 # skip this check for everything but PRs
 if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
   exit 0
@@ -18,7 +20,7 @@ elif [ "$TRAVIS_COMMIT" != "" ]; then
   RANGE=$TRAVIS_COMMIT
 fi
 
-# Travis sends the ranges with 3 dots. Git only wants one.
+# Travis sends the ranges with 3 dots. Git only wants two.
 if [[ "$RANGE" == *...* ]]; then
   RANGE=`echo $TRAVIS_COMMIT_RANGE | sed 's/\.\.\./../'`
 else

CHANGES/5579.feature

@@ -0,0 +1 @@
+Add the ability to set a certification status for a collection version.

CHANGES/5579.removal

@@ -0,0 +1 @@
+Change `is_certified` to `certification` enum on `CollectionVersion`.

CHANGES/5580.misc

@@ -0,0 +1 @@
+Depend on pulpcore, directly, instead of pulpcore-plugin.

pulp_ansible/app/galaxy/v3/serializers.py

@@ -75,7 +75,7 @@ class CollectionVersionListSerializer(serializers.ModelSerializer):
     updated_at = serializers.DateTimeField(source="collection.pulp_last_updated")
 
     class Meta:
-        fields = ("version", "is_certified", "href", "created_at", "updated_at")
+        fields = ("version", "certification", "href", "created_at", "updated_at")
         model = models.CollectionVersion
 
     def get_href(self, obj):

pulp_ansible/app/galaxy/v3/views.py

@@ -238,7 +238,7 @@ def set_certified(self, request, *args, **kwargs):
         Set collection version certified status.
         """
         obj = self.get_object()
-        obj.is_certified = request.method == "PUT"
+        obj.certification = request.data["certification"]
         obj.save()
         return Response({})
 

pulp_ansible/app/migrations/0014_certification_enum.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.2.6 on 2019-10-18 16:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ansible', '0013_pulp_fields'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='collectionversion',
+            name='is_certified',
+        ),
+        migrations.AddField(
+            model_name='collectionversion',
+            name='certification',
+            field=models.CharField(choices=[('certified', 'certified'), ('not_certified', 'not_certified'), ('needs_review', 'needs_review')], default='needs_review', max_length=13),
+        ),
+    ]

pulp_ansible/app/models.py

@@ -112,8 +112,7 @@ class CollectionVersion(Content):
         version (models.CharField): The version of the collection.
         is_highest (models.BooleanField): Indicates that the version is the highest one
             in the collection.
-        is_certified (models.BooleanField): Indicates that the version was approved as
-            certified version.
+        certification (models.CharField): Enum with certification status.
 
     Relations:
 
@@ -123,6 +122,12 @@ class CollectionVersion(Content):
 
     TYPE = "collection_version"
 
+    CERTIFICATION_CHOICES = [
+        ("certified", "certified"),
+        ("not_certified", "not_certified"),
+        ("needs_review", "needs_review"),
+    ]
+
     # Data Fields
     authors = psql_fields.ArrayField(models.CharField(max_length=64), default=list, editable=False)
     contents = psql_fields.JSONField(default=list, editable=False)
@@ -139,7 +144,9 @@ class CollectionVersion(Content):
     version = models.CharField(max_length=32, editable=False)
 
     is_highest = models.BooleanField(editable=False, default=False)
-    is_certified = models.BooleanField(default=False)
+    certification = models.CharField(
+        max_length=13, choices=CERTIFICATION_CHOICES, default="needs_review"
+    )
 
     # Foreign Key Fields
     collection = models.ForeignKey(
@@ -159,6 +166,19 @@ def relative_path(self):
             namespace=self.namespace, name=self.name, version=self.version
         )
 
+    def save(self, *args, **kwargs):
+        """
+        Validates certification before save.
+
+        """
+        certification_choices = [value for key, value in self.CERTIFICATION_CHOICES]
+        if self.certification not in certification_choices:
+            raise ValueError(
+                "Invalid certification value: '{value}'".format(value=self.certification)
+            )
+
+        super().save(*args, **kwargs)
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("namespace", "name", "version")

pulp_ansible/app/serializers.py

@@ -271,7 +271,7 @@ class CollectionVersionSerializer(SingleArtifactContentSerializer, ContentChecks
         help_text=_("The URL to the collection issue tracker."), allow_blank=True, max_length=128
     )
 
-    is_certified = serializers.BooleanField(help_text=_("Indicates that the version is certified"))
+    certification = serializers.CharField(help_text=_("Indicates that the version is certified"))
 
     license = serializers.ListField(
         help_text=_("A list of licenses for content inside of a collection."),
@@ -306,7 +306,7 @@ class Meta:
                 "documentation",
                 "homepage",
                 "issues",
-                "is_certified",
+                "certification",
                 "license",
                 "name",
                 "namespace",

pulp_ansible/app/viewsets.py

@@ -1,10 +1,12 @@
+from collections import defaultdict
 from gettext import gettext as _
+from packaging.version import parse
 
 from django.contrib.postgres.search import SearchQuery
 from django.db import IntegrityError
 from django.db.models import fields as db_fields
 from django.db.models.expressions import F, Func
-from django_filters import filters
+from django_filters import filters, MultipleChoiceFilter
 from drf_yasg.utils import swagger_auto_schema
 from rest_framework import mixins, serializers, viewsets
 from rest_framework.decorators import action
@@ -102,7 +104,8 @@ class CollectionVersionFilter(ContentFilter):
 
     namespace = filters.CharFilter(field_name="namespace")
     name = filters.CharFilter(field_name="name")
-    is_highest = filters.BooleanFilter(field_name="is_highest")
+    is_highest = filters.BooleanFilter(field_name="is_highest", method="get_highest")
+    certification = MultipleChoiceFilter(choices=CollectionVersion.CERTIFICATION_CHOICES)
     q = filters.CharFilter(field_name="q", method="filter_by_q")
     tags = filters.CharFilter(
         field_name="tags",
@@ -150,9 +153,39 @@ def filter_by_tags(self, qs, name, value):
             qs = qs.filter(tags__name=tag)
         return qs
 
+    def get_highest(self, qs, name, value):
+        """
+        Combine certification and is_highest filters.
+
+        If certification and is_highest are used together,
+        get the highest version for the specified certification
+        """
+        certification = self.data.get("certification")
+
+        if not certification:
+            return qs.filter(is_highest=value)
+
+        qs = qs.filter(certification=certification)
+        if not qs.count():
+            return qs
+
+        latest_pks = []
+        namespace_name_dict = defaultdict(lambda: defaultdict(list))
+        for collection in qs.all():
+            version_entry = (parse(collection.version), collection.pk)
+            namespace_name_dict[collection.namespace][collection.name].append(version_entry)
+
+        for namespace, name_dict in namespace_name_dict.items():
+            for name, version_list in name_dict.items():
+                version_list.sort(reverse=True)
+                latest_pk = version_list[0][1]
+                latest_pks.append(latest_pk)
+
+        return qs.filter(pk__in=latest_pks)
+
     class Meta:
         model = CollectionVersion
-        fields = ["namespace", "name", "version", "q", "is_highest", "tags"]
+        fields = ["namespace", "name", "version", "q", "is_highest", "certification", "tags"]
 
 
 class CollectionVersionViewSet(ContentViewSet):