Merge pull request #43 from rajulkumar/dst_3515

Add option to clean lookaside cache for a package

alt_src/alt_src.py

@@ -662,6 +662,13 @@ def delete_local_tag(self, tag):
         cmd.extend(['tag', '-d', tag])
         self.log_cmd(cmd, cwd=self.checkout)
 
+    def clean_lookaside_cache(self):
+        if not self.options.keep_lookaside:
+            lookaside_path = os.path.join(self.options.config['lookaside'],
+                                            self.package, self.options.branch)
+            if os.path.exists(lookaside_path):
+                self.logger.warning("Cleaning lookaside cache: %s", lookaside_path)
+                wipe_dir(lookaside_path)
 
 class Stager(BaseProcessor):
     MMD_DEBRAND_RTYPES = [
@@ -942,7 +949,7 @@ def import_sources(self):
         """Import our source srpm/modulemd on the specified branch"""
         # clear our checkout dir
         dst = self.checkout
-        wipe_git_dir(dst)
+        wipe_dir(dst, git=True)
         # confirm that the directory is wiped, because in the duplicate check
         # below, we want to be sure that the duplicate content really came
         # from our exploded srpm
@@ -967,6 +974,10 @@ def import_sources(self):
                 to_move.append(fname)
         to_move.sort()
 
+        # clean lookaside cache
+        if to_move:
+            self.clean_lookaside_cache()
+
         # move files to lookaside
         meta = open(os.path.join(dst, ".%s.metadata" % self.package), 'w')
         gitignore = open(os.path.join(dst, ".gitignore"), 'w')
@@ -1712,6 +1723,7 @@ def run(self):
         self.push_git(state)
         self.set_state('PUSHED')
         self.notify()
+        self.clean_lookaside_cache()
 
     def check_workdir(self):
         self.workdir = dirname = self.get_workdir()
@@ -2103,9 +2115,9 @@ def relocate_sources(headers, dir):
         os.rename(src, os.path.join(destdir, basename))
 
 
-def wipe_git_dir(dirname):
+def wipe_dir(dirname, git=False):
     for fname in os.listdir(dirname):
-        if fname == '.git':
+        if git and fname == '.git':
             continue
         path = os.path.join(dirname, fname)
         if os.path.isdir(path):
@@ -2196,6 +2208,8 @@ def main(args):
                       help=_("keep lookaside sources in staging checkout"))
     parser.add_option("-o", "--option", dest="copts", action="append", metavar="OPT=VALUE",
                       help=_("set config option"))
+    parser.add_option("--keep-lookaside", action="store_true", default=False,
+                      help=_("keep sources in staging lookaside cache"))
     (options, args) = parser.parse_args(args)
 
     options.branch = args[0]

tests/test_rpms.py

@@ -59,7 +59,8 @@ def pushdir(tempdir):
 
 @pytest.fixture
 def lookasidedir(tempdir):
-    return os.path.join(tempdir, 'lookaside')
+    """path where lookaside content lands post push"""
+    return os.path.join(tempdir, 'lookaside_dest')
 
 
 @pytest.fixture
@@ -177,7 +178,7 @@ def get_test_mmd_str_and_dict():
     ('c7', 'grub2', '2.02', '0.64.el7'),
     ('c7', 'ntp', '4.2.6p5', '25.el7_3.2'),
 ])
-def test_push_with_debrand(config_file, pushdir, lookasidedir,
+def test_push_with_debrand(config_file, default_config, pushdir, lookasidedir,
                            branch, name, version, release, capsys):
     """Verify that alt-src command completes without any errors and generates
     a debranding commit for the given RPM."""
@@ -205,7 +206,12 @@ def test_push_with_debrand(config_file, pushdir, lookasidedir,
         assert_that(rpm not in DEBRAND_XFAIL,
                     'RPM was expected to fail debranding, but passed')
 
-    # lookaside dir should have content
+    # lookaside cache is empty
+    lookaside_cache = '%s/%s/%s' % (default_config['lookaside'], name, branch)
+    files = os.listdir(lookaside_cache)
+    assert_that(files, empty())
+
+    # remote lookaside dir should have content
     lookaside = '%s/%s/%s' % (lookasidedir, name, branch)
     files = os.listdir(lookaside)
     assert_that(files, not_(empty()))
@@ -216,8 +222,8 @@ def test_push_with_debrand(config_file, pushdir, lookasidedir,
     ('c7', 'grub2', '2.02', '0.64.el7'),
     ('c7', 'ntp', '4.2.6p5', '25.el7_3.2'),
 ])
-def test_repush_with_staged_data(config_file, pushdir, lookasidedir,
-                           branch, name, version, release, capsys):
+def test_repush_with_staged_data(config_file, default_config, pushdir, lookasidedir,
+                                 branch, name, version, release, capsys):
     """Push once, the push again. The script should warn, but not error"""
 
     rpm = '-'.join([name, version, release]) +  '.src.rpm'
@@ -258,6 +264,11 @@ def test_repush_with_staged_data(config_file, pushdir, lookasidedir,
     w_expect = '[WARNING] Skipping push for duplicate content'
     assert [l for l in out_lines if w_expect in l]
 
+    # lookaside cache is empty
+    lookaside_cache = '%s/%s/%s' % (default_config['lookaside'], name, branch)
+    files = os.listdir(lookaside_cache)
+    assert_that(files, empty())
+
     # lookaside dir should have content
     lookaside = '%s/%s/%s' % (lookasidedir, name, branch)
     files = os.listdir(lookaside)
@@ -536,6 +547,11 @@ def test_repush_with_state_staged(config_file, pushdir, lookasidedir, default_co
     with patch("alt_src.alt_src.Pusher.push_git", autospec=True, side_effect=RuntimeError):
         assert_that(calling(main).with_args(options), exits(2))
 
+    # lookaside cache has content after stage
+    lookaside_cache = '%s/%s/%s' % (default_config['lookaside'], 'grub2', 'c7')
+    files = os.listdir(lookaside_cache)
+    assert_that(files, not_(empty()))
+
     # before push again, we need to change the branch to some unkown branch
     # to simulate what would happen in push_git
     cmd = 'git checkout -b arandombranch'
@@ -557,6 +573,11 @@ def test_repush_with_state_staged(config_file, pushdir, lookasidedir, default_co
 %s/c7/rpms/g/grub2/grub2-2.02-0.64.el7' % default_config['stagedir']
     assert [l for l in out_lines if expected in l]
 
+    # lookaside cache is empty after push completes
+    lookaside_cache = '%s/%s/%s' % (default_config['lookaside'], 'grub2', 'c7')
+    files = os.listdir(lookaside_cache)
+    assert_that(files, empty())
+
     # lookaside dir should have content
     lookaside = '%s/%s/%s' % (lookasidedir, 'grub2', 'c7')
     files = os.listdir(lookaside)
@@ -738,7 +759,7 @@ def test_acquire_release_lock(tempdir):
     remove_handlers()
 
 
-def test_stage_only(config_file, pushdir, capsys):
+def test_stage_only(config_file, default_config, pushdir, capsys):
     """
     test a task without push option
     """
@@ -756,6 +777,11 @@ def test_stage_only(config_file, pushdir, capsys):
     assert_that(len(err), equal_to(0))
     files = os.listdir(pushdir)
     assert_that(files, empty())
+
+    #lookaside cache has content
+    lookaside_cache = '%s/%s/%s' % (default_config['lookaside'], 'grub2', 'c7')
+    files = os.listdir(lookaside_cache)
+    assert_that(files, not_(empty()))
     remove_handlers()
 
 
@@ -981,6 +1007,73 @@ def test_stage_module_src(config_file, pushdir, lookasidedir, capsys, default_co
     remove_handlers()
 
 
+def test_initial_lookaside_cleanup(config_file, default_config):
+    """Files present previously in the lookaside cache is cleared."""
+    name = 'ntp'
+    rpm = 'ntp-4.2.6p5-25.el7_3.2.src.rpm'
+    branch = 'c7'
+
+    lookaside_cache = '%s/%s/%s' % (default_config['lookaside'], name, branch)
+    os.makedirs(lookaside_cache)
+
+    # lookaside is not empty initially
+    # assuming content from some previous request
+    f = tempfile.NamedTemporaryFile(dir=lookaside_cache)
+    files = os.listdir(lookaside_cache)
+    prev_file = files[0]
+    assert_that(files, not_(empty()))
+
+    options = ['-v',
+               '-c', config_file,
+               branch,
+               os.path.join(RPMS_PATH, rpm)
+    ]
+    assert_that(calling(main).with_args(options), exits(0))
+
+    # lookaside cache has files after staging
+    # but files from the previous push were cleared
+    files = os.listdir(lookaside_cache)
+    assert_that(files, not_(empty()))
+    assert prev_file not in files
+    remove_handlers()
+
+
+def test_keep_lookaside(config_file, lookasidedir, default_config, capsys):
+    """Files in lookaside cache are not cleared once the push completes
+       when using keep-lookaside option."""
+    name = 'ntp'
+    rpm = 'ntp-4.2.6p5-25.el7_3.2.src.rpm'
+    branch = 'c7'
+
+    options = ['-v',
+               '-c', config_file,
+               '--keep-lookaside',
+               branch,
+               os.path.join(RPMS_PATH, rpm)
+    ]
+    assert_that(calling(main).with_args(options), exits(0))
+
+    #restaging skips copying to lookaside if files already exist
+    options.append("--restage")
+    assert_that(calling(main).with_args(options), exits(0))
+    out, _ = capsys.readouterr()
+    assert "Skipping source, already in digest" in out
+
+    options.append("--push")
+    assert_that(calling(main).with_args(options), exits(0))
+
+    #lookaside cache has files after the push completes
+    lookaside_cache = '%s/%s/%s' % (default_config['lookaside'], name, branch)
+    files = os.listdir(lookaside_cache)
+    assert_that(files, not_(empty()))
+
+    #lookaside destination has files
+    lookaside = '%s/%s/%s' % (lookasidedir, name, branch)
+    remote_files = os.listdir(lookaside)
+    assert_that(remote_files, not_(empty()))
+    remove_handlers()
+
+
 def test_push_to_pagure(config_file, key_file, pushdir, lookasidedir, capsys):
 
     rpm = 'grub2-2.02-0.64.el7.src.rpm'