Add generic checks for whether a full sync

should be forced to the repo controller.
closes #1983

docs/dev-guide/integration/rest-api/repo/retrieval.rst

@@ -324,7 +324,6 @@ will either be empty (no importer configured) or contain a single entry.
         "last_sync": "2015-11-06T10:38:23Z",
         "repo_id": "zoo",
         "scratchpad": {
-            "previous_skip_list": [],
             "repomd_revision": 1331832478
         }
     }
@@ -362,7 +361,6 @@ Retrieves the given :term:`importer` (if any) associated with a repository.
     "last_sync": "2015-11-06T10:38:23Z",
     "repo_id": "zoo",
     "scratchpad": {
-        "previous_skip_list": [],
         "repomd_revision": 1331832478
     }
  }

docs/user-guide/release-notes/2.11.x.rst

@@ -0,0 +1,12 @@
+=======================
+Pulp 2.11 Release Notes
+=======================
+
+Pulp 2.11.0
+===========
+
+New Features
+------------
+
+* For RPM content, a full sync will be forced if the sync configuration has been changed or content
+  has been removed since the last sync.

server/pulp/plugins/conduits/mixins.py

@@ -289,9 +289,7 @@ def set_scratchpad(self, value):
         :raises ImporterConduitException: wraps any exception that may occur in the Pulp server
         """
         try:
-            importer = model.Importer.objects.get(repo_id=self.repo_id)
-            importer.scratchpad = value
-            importer.save()
+            model.Importer.objects.get(repo_id=self.repo_id).update(set__scratchpad=value)
         except Exception, e:
             _logger.exception(_('Error setting scratchpad for repo [%(r)s]') % {'r': self.repo_id})
             raise ImporterConduitException(e), None, sys.exc_info()[2]

server/pulp/server/controllers/repository.py

@@ -798,6 +798,10 @@ def sync(repo_id, sync_config_override=None, scheduled_call_id=None):
                                                                sync_start_timestamp, summary,
                                                                details, result_code,
                                                                before_sync_unit_count)
+        # Update the override config if it has changed
+        if check_override_config_change(repo_id, call_config):
+            model.Importer.objects(repo_id=repo_id).\
+                update(set__last_override_config=call_config.override_config)
         # Do an update instead of a save in case the importer has changed the scratchpad
         model.Importer.objects(repo_id=repo_obj.repo_id).update(set__last_sync=sync_end_timestamp)
         # Add a sync history entry for this run
@@ -816,6 +820,124 @@ def sync(repo_id, sync_config_override=None, scheduled_call_id=None):
     return TaskResult(sync_result, spawned_tasks=spawned_tasks)
 
 
+def check_unit_removed_since_last_sync(conduit, repo_id):
+    """
+    Checks whether a content unit has been removed since the last_sync timestamp.
+
+    :param conduit: allows the plugin to interact with core pulp
+    :type  conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
+    :param repo_id: identifies the repo to sync
+    :type  repo_id: str
+
+    :return:  Whether a content unit has been removed since the last_sync timestamp
+    :rtype:   bool
+    """
+    last_sync = conduit.last_sync()
+
+    if last_sync is None:
+        return False
+
+    # convert the iso8601 datetime string to a python datetime object
+    last_sync = dateutils.parse_iso8601_datetime(last_sync)
+    repo_obj = model.Repository.objects.get_repo_or_missing_resource(repo_id=repo_id)
+    last_removed = repo_obj.last_unit_removed
+
+    # check if a unit has been removed since the past sync
+    if last_removed is not None:
+        if last_removed > last_sync:
+            return True
+
+    return False
+
+
+def check_config_updated_since_last_sync(conduit, repo_id):
+    """
+    Checks whether the config has been changed since the last sync occurred.
+
+    :param conduit: allows the plugin to interact with core pulp
+    :type  conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
+    :param repo_id: identifies the repo to sync
+    :type  repo_id: str
+
+    :return:  Whether the config has been changed since the last sync occurred.
+    :rtype:   bool
+    """
+    last_sync = conduit.last_sync()
+
+    if last_sync is None:
+        return False
+
+    # convert the iso8601 datetime string to a python datetime object
+    last_sync = dateutils.parse_iso8601_datetime(last_sync)
+    repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id)
+    # the timestamp of the last configuration change
+    last_updated = repo_importer.last_updated
+
+    # check if a configuration change occurred after the most recent sync
+    if last_updated is not None:
+        if last_sync < last_updated:
+            return True
+
+    return False
+
+
+def check_override_config_change(repo_id, call_config):
+    """
+    Checks whether the override config is different from the override config on
+    the previous sync.
+
+    :param repo_id:     identifies the repo to sync
+    :type  repo_id:     str
+    :param call_config: Plugin Call Configuration
+    :type  call_config: pulp.plugins.config.PluginCallConfiguration
+
+    :return:  Whether the override config is different from the override config on
+    the previous sync.
+    :rtype:   bool
+    """
+    repo_importer = model.Importer.objects.get_or_404(repo_id=repo_id)
+
+    # check if the override config is different from the last override config,
+    # excluding the 'force_full' key.  otherwise using the --force-full flag
+    # would always trigger a full sync on the next sync too
+    prev_config = repo_importer.last_override_config.copy()
+    current_config = call_config.override_config.copy()
+
+    prev_config.pop('force_full', False)
+    current_config.pop('force_full', False)
+
+    return prev_config != current_config
+
+
+def check_perform_full_sync(repo_id, conduit, call_config):
+    """
+    Performs generic checks to determine if the sync should be a "full" sync.
+    Checks if the "force full" flag has been set, if content has been removed
+    since the last sync, and whether the configuration has changed since the
+    last sync (including override configs).
+
+    Plugins may want to perform additional checks beyond the ones appearing here.
+
+    :param repo_id:     identifies the repo to sync
+    :type  repo_id:     str
+    :param conduit: allows the plugin to interact with core pulp
+    :type  conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
+    :param call_config: Plugin Call Configuration
+    :type  call_config: pulp.plugins.config.PluginCallConfiguration
+
+    :return:  Whether a full sync needs to be performed
+    :rtype:  bool
+    """
+    force_full = call_config.get('force_full', False)
+    first_sync = conduit.last_sync() is None
+
+    content_removed = check_unit_removed_since_last_sync(conduit, repo_id)
+    config_changed = check_config_updated_since_last_sync(conduit, repo_id)
+    override_config_changed = check_override_config_change(repo_id, call_config)
+
+    return force_full or first_sync or content_removed or config_changed or override_config_changed
+
+
 def _reposync_result(repo, importer, sync_start, summary, details, result_code, initial_unit_count):
     """
     Creates repo sync result.

server/pulp/server/db/migrations/0025_importer_schema_change.py

@@ -0,0 +1,23 @@
+from pulp.common import dateutils
+from pulp.server.db.connection import get_collection
+
+
+def migrate(*args, **kwargs):
+    """
+    Add last_updated and last_override_config to the importer collection.
+    """
+    updated_key = 'last_updated'
+    config_key = 'last_override_config'
+    collection = get_collection('repo_importers')
+    for importer in collection.find():
+        if config_key not in importer.keys():
+            importer[config_key] = {}
+
+        if updated_key in importer.keys():
+            continue
+        elif 'last_sync' in importer.keys():
+            importer[updated_key] = dateutils.parse_iso8601_datetime(importer['last_sync'])
+        else:
+            importer[updated_key] = dateutils.now_utc_datetime_with_tzinfo()
+
+        collection.save(importer)

server/pulp/server/db/model/__init__.py

@@ -235,6 +235,8 @@ class Importer(AutoRetryDocument):
     config = DictField()
     scratchpad = DictField(default=None)
     last_sync = ISO8601StringField()
+    last_updated = UTCDateTimeField()
+    last_override_config = DictField()
 
     # For backward compatibility
     _ns = StringField(default='repo_importers')
@@ -260,6 +262,18 @@ def pre_delete(cls, sender, document, **kwargs):
             repo=document.repo_id))
         query_set.delete()
 
+    @classmethod
+    def pre_save(cls, sender, document, **kwargs):
+        """
+        The signal that is triggered before importer is saved.
+
+        :param sender:   class of sender (unused)
+        :type sender:    object
+        :param document: mongoengne document being saved
+        :type document:  pulp.server.db.model.Importer
+        """
+        document.last_updated = dateutils.now_utc_datetime_with_tzinfo()
+
     def delete(self):
         """
         Delete the Importer. Remove any documents it has stored.
@@ -348,6 +362,7 @@ def _write_pem_file(self, config_key, path):
 
 
 signals.pre_delete.connect(Importer.pre_delete, sender=Importer)
+signals.pre_save.connect(Importer.pre_save, sender=Importer)
 
 
 class ReservedResource(AutoRetryDocument):

server/test/unit/plugins/conduits/test_mixins.py

@@ -383,8 +383,9 @@ def test_get_scratchpad(self, mock_importer_qs):
     def test_set_scratchpad(self, mock_importer_qs):
         mock_importer = mock_importer_qs.get.return_value
         self.mixin.set_scratchpad('foo')
-        mock_importer.save.assert_called_once_with()
-        self.assertEqual(mock_importer.scratchpad, 'foo')
+        mock_importer.update.assert_called_once_with(set__scratchpad='foo')
+        sp = self.mixin.get_scratchpad()
+        self.assertTrue(sp is mock_importer_qs.get.return_value.scratchpad)
 
     def test_set_scratchpad_server_error(self, mock_importer_qs):
         mock_importer_qs.get.side_effect = Exception()