Problem: API does not support publishing a repository

Solution: Add API endpoint to dispatch a publish with a specific publisher.

closes #2398

platform/pulp/app/tasks/publisher.py

@@ -1,6 +1,9 @@
+from gettext import gettext as _
+
 from celery import shared_task
 
 from pulp.app import models
+from pulp.tasking.services import storage
 from pulp.tasking.tasks import UserFacingTask
 
 
@@ -15,3 +18,22 @@ def delete(repo_name, publisher_name):
     :type  publisher_name:  str
     """
     models.Publisher.objects.filter(name=publisher_name, repository__name=repo_name).delete()
+
+@shared_task(base=UserFacingTask)
+def publish(repo_name, publisher_name):
+    """
+    Call publish on the publisher defined by a plugin.
+
+    A working directory is prepared, the plugin's publish is called, and then working directory is
+    removed.
+
+    Args:
+        repo_name (str): unique name to specify the repository.
+        publisher_name (str): name to specify the Publisher.
+    """
+    publisher = models.Publisher.objects.get(name=publisher_name,
+                                             repository__name=repo_name).cast()
+
+    with storage.working_dir_context() as working_dir:
+        publisher.working_dir = working_dir
+        publisher.publish()

platform/pulp/app/viewsets/repository.py

@@ -150,6 +150,16 @@ def destroy(self, request, pk):
 
         return OperationPostponedResponse([async_result])
 
+    @decorators.detail_route()
+    def publish(self, request, pk):
+        publisher = self.get_object()
+        async_result = tasks.publisher.publish.apply_async_with_reservation(
+            tags.RESOURCE_REPOSITORY_TYPE, publisher.repository.name,
+            kwargs={'repo_name': publisher.repository.name,
+                    'publisher_name': publisher.name}
+        )
+        return OperationPostponedResponse([async_result])
+
 
 class RepositoryContentViewSet(NamedModelViewSet):
     endpoint_name = 'repositorycontents'

platform/pulp/plugin/models/publisher.py

@@ -20,53 +20,23 @@ class Publisher(PlatformPublisher):
     add additional persistent publisher data by subclassing this object and adding Django
     fields. We defer to the Django docs on extending this model definition with additional fields.
 
-    Validation is done the Django way, so custom validation can also be added the same as any
-    Django model. We defer to the Django docs on adding custom validation to the subclassed
-    publisher. If any of the model validation methods are overridden, be sure to call super() so
-    the platform can still perform its validation.
-
-    Instantiation and calling of the subclassed plugin Publisher is described in detail in the
-    :meth: `Publisher.publish` method.
+    Validation of the publisher is done at the API level by a plugin defined subclass of
+    :class: `pulp.plugin.serializers.repository.PublisherSerializer`.
     """
 
     def publish(self):
         """
-        Perform a publish
+        Perform a publish.
 
         It is expected that plugins wanting to support publish will provide an implementation on the
         subclassed Publisher.
 
         The model attributes encapsulate all of the information required to publish. This includes
-        the platform :class: `pulp.app.models.Publish` base attributes and any custom
-        attributes defined by the subclass.
-
-        The model attributes were loaded from the database and then had the user specified override
-        config applied on top. Additionally the publisher is read-only and prevents the saving of
-        changes to the Publisher instance.
-
-        Instantiation and calling of the publish method by the platform is roughly done with the
-        following:
-
-            1. The plugin provides an implementation called WidgetPublisher which subclasses
-               Publisher
-
-            2. The user makes a call to publish widget_publisher (say id=10) with some override
-               config
-
-            3. The platform loads the saved
-                >>> wd = WidgetPublisher.objects.get(id=10)
-
-            4. The platform puts the WidgetPublisher into read-only mode
-
-            5. The override config values are written over the in memory WidgetPublisher
-
-            6. Call the full_clean() method on the Django model for validation
-
-                >>> wd.full_clean()
-
-            7. Call into the publish method
+        the platform :class: `pulp.app.models.Publisher` base attributes and any custom attributes
+        defined by the subclass.
 
-                >>> wd.publish()
+        Instantiation and calling of the publish method by the platform is defined by
+        :meth: `pulp.app.tasks.publisher.publish`.
 
         Subclasses are designed to override this default implementation and should not call super().
         """

server/pulp/server/controllers/repository.py

@@ -800,161 +800,6 @@ def sync_history(start_date, end_date, repo_id):
     return RepoSyncResult.get_collection().find(search_params)
 
 
-@celery.task(base=PulpTask)
-def queue_publish(repo_id, distributor_id, overrides=None, scheduled_call_id=None):
-    """
-    Queue a repo publish task.
-
-    :param repo_id: id of the repo to publish
-    :type  repo_id: str
-    :param distributor_id: publish the repo with this distributor
-    :type  distributor_id: str
-    :param overrides: dictionary of options to pass to the publish task
-    :type  overrides: dict or None
-    :param scheduled_call_id: id of scheduled call that dispatched this task
-    :type  scheduled_call_id: str
-    """
-    kwargs = {'repo_id': repo_id, 'dist_id': distributor_id, 'publish_config_override': overrides,
-              'scheduled_call_id': scheduled_call_id}
-    tags = [resource_tag(RESOURCE_REPOSITORY_TYPE, repo_id),
-            action_tag('publish')]
-    publish.apply_async_with_reservation(RESOURCE_REPOSITORY_TYPE, repo_id, tags=tags,
-                                         kwargs=kwargs)
-
-
-@celery.task(base=UserFacingTask, name='pulp.server.managers.repo.publish.publish')
-def publish(repo_id, dist_id, publish_config_override=None, scheduled_call_id=None):
-    """
-    Uses the given distributor to publish the repository.
-
-    The publish operation is executed synchronously in the caller's thread and will block until it
-    is completed. The caller must take the necessary steps to address the fact that a publish call
-    may be time intensive.
-
-    :param repo_id: identifies the repo being published
-    :type  repo_id: str
-    :param dist_id: identifies the repo's distributor to publish
-    :type  dist_id: str
-    :param publish_config_override: optional config values to use for this publish call only
-    :type  publish_config_override: dict, None
-    :param scheduled_call_id: id of scheduled call that dispatched this task
-    :type  scheduled_call_id: str
-
-    :return: report of the details of the publish
-    :rtype:  pulp.server.db.model.repository.RepoPublishResult
-
-    :raises pulp_exceptions.MissingResource: if distributor/repo pair does not exist
-    """
-    repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id)
-    dist = model.Distributor.objects.get_or_404(repo_id=repo_id, distributor_id=dist_id)
-    dist_inst, dist_conf = _get_distributor_instance_and_config(repo_id, dist_id)
-
-    # Assemble the data needed for the publish
-    conduit = RepoPublishConduit(repo_id, dist_id)
-    call_config = PluginCallConfiguration(dist_conf, dist.config, publish_config_override)
-    transfer_repo = repo_obj.to_transfer_repo()
-    transfer_repo.working_dir = get_working_directory()
-
-    # Fire events describing the publish state
-    fire_manager = manager_factory.event_fire_manager()
-    fire_manager.fire_repo_publish_started(repo_id, dist_id)
-    result = check_publish(repo_obj, dist_id, dist_inst, transfer_repo, conduit, call_config)
-    fire_manager.fire_repo_publish_finished(result)
-    return result
-
-
-def check_publish(repo_obj, dist_id, dist_inst, transfer_repo, conduit, call_config):
-    """
-    Check if the publish should be a no operation and therefore skipped.
-
-    :param repo_obj: repository object
-    :type  repo_obj: pulp.server.db.model.Repository
-    :param dist_id: identifies the distributor
-    :type  dist_id: str
-    :param dist_inst: instance of the distributor
-    :type  dist_inst: dict
-    :param transfer_repo: dict representation of a repo for the plugins to use
-    :type  transfer_repo: pulp.plugins.model.Repository
-    :param conduit: allows the plugin to interact with core pulp
-    :type  conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
-    :param call_config: allows the plugin to retrieve values
-    :type  call_config: pulp.plugins.config.PluginCallConfiguration
-
-    :return: publish result containing information about the publish
-    :rtype:  pulp.server.db.model.repository.RepoPublishResult
-    """
-    force_full = call_config.get('force_full', False)
-    config_override = call_config.override_config
-    last_published = conduit.last_publish()
-    last_unit_removed = repo_obj.last_unit_removed
-    dist = model.Distributor.objects.get_or_404(repo_id=repo_obj.repo_id,
-                                                distributor_id=dist_id)
-    if last_published:
-        the_timestamp = dateutils.format_iso8601_datetime(last_published)
-        last_updated = model.RepositoryContentUnit.objects(repo_id=repo_obj.repo_id,
-                                                           updated__gte=the_timestamp).count()
-        units_removed = last_unit_removed is not None and last_unit_removed > last_published
-        dist_updated = dist.last_updated > last_published
-    else:
-        published_after_predistributor = False
-    predistributor_id = call_config.get('predistributor_id')
-    if predistributor_id:
-        predistributor = model.Distributor.objects.get_or_404(repo_id=repo_obj.repo_id,
-                                                              distributor_id=predistributor_id)
-        predistributor_last_published = predistributor["last_publish"]
-        if predistributor_last_published and last_published:
-            published_after_predistributor = last_published > predistributor_last_published
-
-    same_override = dist.last_override_config == config_override
-    if not same_override:
-        # Use raw pymongo not to fire the signal hander
-        model.Distributor.objects(
-            repo_id=repo_obj.repo_id,
-            distributor_id=dist_id).update(set__last_override_config=config_override)
-
-    # Check if a predistributor is configured and the predistributor has not published since the
-    # last publish.
-    skip_for_predistributor = (predistributor_id and (published_after_predistributor or
-                                                      not predistributor_last_published))
-    # Check if content has not changed since last publish and a predistributor is not defined.
-    unchanged_content_and_no_predistributor = (last_published and not last_updated and
-                                               not units_removed and not predistributor_id)
-    # We want to skip based on predistributor conditions. We also want to skip if repository
-    # content has not changed since last publish and no predistributor is defined. We want to not
-    # skip if 'force_full' is configured or the distributor config has changed since last publish.
-    if (skip_for_predistributor and not last_published) or\
-            (last_published and not force_full and not dist_updated and same_override and
-                (skip_for_predistributor or unchanged_content_and_no_predistributor)):
-
-        publish_result_coll = RepoPublishResult.get_collection()
-        publish_start_timestamp = _now_timestamp()
-        publish_end_timestamp = _now_timestamp(string=False)
-
-        # Use raw pymongo not to fire the signal hander
-        model.Distributor.objects(
-            repo_id=repo_obj.repo_id,
-            distributor_id=dist_id).update(set__last_publish=publish_end_timestamp)
-
-        result_code = RepoPublishResult.RESULT_SKIPPED
-        _logger.debug('publish skipped for repo [%s] with distributor ID [%s]' % (
-                      repo_obj.repo_id, dist_id))
-
-        if predistributor_id:
-            reason = _('Predistributor %(predistributor_id)s has not published since last '
-                       'publish.') % {'predistributor_id': predistributor_id}
-        else:
-            reason = _('Repository content has not changed since last publish.')
-        result = RepoPublishResult.skipped_result(
-            repo_obj.repo_id, dist.distributor_id, dist.distributor_type_id,
-            publish_start_timestamp, publish_end_timestamp, result_code, reason)
-        publish_result_coll.save(result)
-
-    else:
-        result = _do_publish(repo_obj, dist_id, dist_inst, transfer_repo, conduit, call_config)
-
-    return result
-
-
 def _get_distributor_instance_and_config(repo_id, distributor_id):
     """
     For a given repository and distributor, retrieve the instance of the distributor and its
@@ -973,85 +818,6 @@ def _get_distributor_instance_and_config(repo_id, distributor_id):
     return distributor, config
 
 
-def _do_publish(repo_obj, dist_id, dist_inst, transfer_repo, conduit, call_config):
-    """
-    Publish the repository using the given distributor.
-
-    :param repo_obj: repository object
-    :type  repo_obj: pulp.server.db.model.Repository
-    :param dist_id: identifies the distributor
-    :type  dist_id: str
-    :param dist_inst: instance of the distributor
-    :type  dist_inst: dict
-    :param transfer_repo: dict representation of a repo for the plugins to use
-    :type  transfer_repo: pulp.plugins.model.Repository
-    :param conduit: allows the plugin to interact with core pulp
-    :type  conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
-    :param call_config: allows the plugin to retrieve values
-    :type  call_config: pulp.plugins.config.PluginCallConfiguration
-
-    :return: publish result containing information about the publish
-    :rtype:  pulp.server.db.model.repository.RepoPublishResult
-
-    :raises pulp_exceptions.PulpCodedException: if the publish report's success flag is falsey
-    """
-    publish_result_coll = RepoPublishResult.get_collection()
-    publish_start_timestamp = _now_timestamp()
-    try:
-        # Add the register_sigterm_handler decorator to the publish_repo call, so that we can
-        # respond to signals by calling the Distributor's cancel_publish_repo() method.
-        publish_repo = register_sigterm_handler(dist_inst.publish_repo,
-                                                dist_inst.cancel_publish_repo)
-        publish_report = publish_repo(transfer_repo, conduit, call_config)
-        if publish_report is not None and hasattr(publish_report, 'success_flag') \
-                and not publish_report.success_flag:
-            _logger.info(publish_report.summary)
-            raise pulp_exceptions.PulpCodedException(
-                error_code=error_codes.PLP0034, repo_id=repo_obj.repo_id,
-                distributor_id=dist_id, summary=publish_report.summary
-            )
-
-    except Exception, e:
-        exception_timestamp = _now_timestamp()
-
-        # Reload the distributor in case the scratchpad is set by the plugin
-        dist = model.Distributor.objects.get_or_404(repo_id=repo_obj.repo_id,
-                                                    distributor_id=dist_id)
-        # Add a publish history entry for the run
-        result = RepoPublishResult.error_result(
-            repo_obj.repo_id, dist.distributor_id, dist.distributor_type_id,
-            publish_start_timestamp, exception_timestamp, e, sys.exc_info()[2])
-        publish_result_coll.save(result)
-
-        _logger.exception(
-            _('Exception caught from plugin during publish for repo [%(r)s]'
-              % {'r': repo_obj.repo_id}))
-        raise
-
-    publish_end_timestamp = _now_timestamp(string=False)
-
-    # Reload the distributor in case the scratchpad is set by the plugin
-    dist = model.Distributor.objects.get_or_404(
-        repo_id=repo_obj.repo_id, distributor_id=dist_id)
-
-    if not publish_report.canceled_flag:
-        # Use raw pymongo not to fire the signal hander
-        model.Distributor.objects(repo_id=repo_obj.repo_id, distributor_id=dist_id).\
-            update(set__last_publish=publish_end_timestamp)
-
-    # Add a publish entry
-    summary = publish_report.summary
-    details = publish_report.details
-    _logger.debug('publish succeeded for repo [%s] with distributor ID [%s]' % (
-                  repo_obj.repo_id, dist_id))
-    result_code = RepoPublishResult.RESULT_SUCCESS
-    result = RepoPublishResult.expected_result(
-        repo_obj.repo_id, dist.distributor_id, dist.distributor_type_id,
-        publish_start_timestamp, publish_end_timestamp, summary, details, result_code)
-    publish_result_coll.save(result)
-    return result
-
-
 def publish_history(start_date, end_date, repo_id, distributor_id):
     """
     Returns a cursor containing the publish history entries for the given repo and distributor.