996198 - CDS sync should not attempt to sync older packages which hav…

…e been removed from RHUA by way of 'num_old_packages_keep'

Partial fix for bz 996198
 - This commit adds ability for CDS to read config file and not sync older packages
 - Future commit will expand this, so the RHUA will send down it's sync options, allowing the CDS config to override anything from RHUA

etc/pulp/cds.conf

@@ -27,3 +27,8 @@ sync_threads = 10
 # Verify existing package options
 verify_checksum = false
 verify_size = false
+# True/False to flag if we should remove older rpms
+remove_old_versions: false
+# Integer to specify how many old packages to keep.
+# Only used if 'removeold' is set to True
+num_old_pkgs_keep: 2

src/pulp/cds/cdslib.py

@@ -313,6 +313,8 @@ def _sync_repo(self, base_url, repo):
         num_threads = self.config.get('cds', 'sync_threads')
         content_base = self.config.get('cds', 'packages_dir')
         packages_dir = os.path.join(content_base, 'packages')
+        remove_old = self.config.getboolean('cds', 'remove_old_versions')
+        num_old_pkgs_keep = self.config.getint('cds', 'num_old_pkgs_keep')
 
         url = '%s/%s' % (base_url, repo['relative_path'])
         log.info('Synchronizing repo at [%s]' % url)
@@ -355,10 +357,12 @@ def _sync_repo(self, base_url, repo):
         verify_options["checksum"] = self.config.getboolean('cds', "verify_checksum")
         fetch = YumRepoGrinder('', url, num_threads, sslverify=ssl_verify,
                                cacert=feed_ca, clicert=feed_cert,
-                               packages_location=packages_dir)
-        fetch.fetchYumRepo(repo_path, verify_options=verify_options)
+                               packages_location=packages_dir,
+                               remove_old=remove_old, numOldPackages=num_old_pkgs_keep)
+        report = fetch.fetchYumRepo(repo_path, verify_options=verify_options)
 
         log.info('Successfully finished synccing [%s]' % url)
+        return report
 
     def _delete_removed_repos(self, repos):
         '''

test/unit/test_cds_plugin.py

@@ -44,6 +44,10 @@ def setUp(self):
             self.config.add_section('cds')
         self.config.set('cds', 'packages_dir', os.path.join(ROOTDIR, 'packages'))
         self.config.set('cds', 'sync_threads', '3')
+        self.config.set('cds', 'verify_size', 'false')
+        self.config.set('cds', 'verify_checksum', 'false')
+        self.config.set('cds', 'remove_old_versions', 'false')
+        self.config.set('cds', 'num_old_pkgs_keep', '0') # value will be ignored when remove_old_versions=false
         self.cds = CdsLib(self.config)
 
         if os.path.exists(TEST_STORAGE_FILE):
@@ -83,3 +87,37 @@ def test_secret(self):
         self.assertEqual(uuid, s)
         secret.delete()
         self.assertFalse(os.path.exists(path))
+
+    def test_basic_sync(self):
+        base_url = "http://jmatthews.fedorapeople.org"
+        repo = {
+            "name":"cdsplugin_test_basic_sync_repo_name",
+            "id":"cdsplugin_test_basic_sync_repo_id",
+            "relative_path":"repo_multiple_versions"
+        }
+        report = self.cds._sync_repo(base_url, repo)
+        self.assertEqual(report.downloads, 12)
+
+    def test_sync_with_remove_old_packages(self):
+        base_url = "http://jmatthews.fedorapeople.org"
+        repo = {
+            "name":"cdsplugin_test_basic_sync_repo_name",
+            "id":"cdsplugin_test_basic_sync_repo_id",
+            "relative_path":"repo_multiple_versions"
+        }
+        config = testutil.load_test_config()
+        if not config.has_section('cds'):
+            config.add_section('cds')
+        config.set('cds', 'packages_dir', os.path.join(ROOTDIR, 'packages'))
+        config.set('cds', 'sync_threads', '3')
+        config.set('cds', 'verify_size', 'false')
+        config.set('cds', 'verify_checksum', 'false')
+        config.set('cds', 'remove_old_versions', 'true')
+        config.set('cds', 'num_old_pkgs_keep', '2')
+        cds = CdsLib(config)
+        report = cds._sync_repo(base_url, repo)
+        # Repo has 12 versions of the same package, we expect to 
+        # Sync latest versions + 'num_old_pkgs_keep'
+        # this means we expect 3 packages to be synced
+        self.assertEqual(report.downloads, 3)
+