Skip to content

Commit

Permalink
Move _sanity_check_repo() to the RPMComposerThread subclass
Browse files Browse the repository at this point in the history 
_sanity_check_repo() was designed to check Yum repositories, but
it was in the PungiComposerThread class which caused it to be run
on modular repositories as well. This commit moves it to the
RPMComposerThread class so that it is only run on Yum repositories
and provides a no-op method on the superclass so that the module
composing class can add a module sanity check later if desired.

fixes fedora-infra#2631

Signed-off-by: Randy Barlow <randy@electronsweatshop.com>
  • Loading branch information
@bowlofeggs
bowlofeggs committed Oct 2, 2018
1 parent 6da52d1 commit e51f6d2
Show file tree
Hide file tree
Showing 5 changed files with 114 additions and 105 deletions.
151 changes: 77 additions & 74 deletions bodhi/server/consumers/masher.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1118,80 +1118,7 @@ def _toss_out_repo(self):
self.save_state()

def _sanity_check_repo(self):
"""Sanity check our repo.
- make sure we didn't compose a repo full of symlinks
- sanity check our repodata
This basically checks that pungi was run with gather_method='hardlink-or-copy' so that
we get a repository with either hardlinks or copied files.
This means that we when we go and sync generated repositories out, we do not need to take
special case to copy the target files rather than symlinks.
"""
self.log.info("Running sanity checks on %s" % self.path)

try:
arches = os.listdir(os.path.join(self.path, 'compose', 'Everything'))
except Exception:
self.log.exception('Empty compose folder? Compose thrown out')
self._toss_out_repo()
raise

if len(arches) == 0:
self.log.error('Empty compose, compose thrown out')
self._toss_out_repo()
raise Exception('Empty compose found')

for arch in arches:
# sanity check our repodata
try:
if arch == 'source':
repodata = os.path.join(self.path, 'compose',
'Everything', arch, 'tree', 'repodata')
sanity_check_repodata(repodata, source=True)
else:
repodata = os.path.join(self.path, 'compose',
'Everything', arch, 'os', 'repodata')
sanity_check_repodata(repodata, source=False)
except Exception:
self.log.exception("Repodata sanity check failed, compose thrown out")
self._toss_out_repo()
raise

# make sure that pungi didn't symlink our packages
try:
if arch == 'source':
dirs = [('tree', 'Packages')]
else:
dirs = [('debug', 'tree', 'Packages'), ('os', 'Packages')]

# Example of full path we are checking:
# self.path/compose/Everything/os/Packages/s/something.rpm
for checkdir in dirs:
checkdir = os.path.join(self.path, 'compose', 'Everything', arch, *checkdir)
subdirs = os.listdir(checkdir)
# subdirs is the self.path/compose/Everything/os/Packages/{a,b,c,...}/ dirs
#
# Let's check the first file in each subdir. If they are correct, we'll assume
# the rest is correct
# This is to avoid tons and tons of IOPS for a bunch of files put in in the
# same way
for subdir in subdirs:
for checkfile in os.listdir(os.path.join(checkdir, subdir)):
if not checkfile.endswith('.rpm'):
continue
if os.path.islink(os.path.join(checkdir, subdir, checkfile)):
self.log.error('Pungi out directory contains at least one '
'symlink at %s', checkfile)
raise Exception('Symlinks found')
# We have checked the first rpm in the subdir
break
except Exception:
self.log.exception('Unable to check pungi mashed repositories, compose thrown out')
self._toss_out_repo()
raise

return True
"""Each Subclass can override this method if they wish to verify the produced repository."""

def _stage_repo(self):
"""Symlink our updates repository into the staging directory."""
Expand Down Expand Up @@ -1356,6 +1283,82 @@ def _copy_additional_pungi_files(self, pungi_conf_dir, template_env):
with open(os.path.join(pungi_conf_dir, 'variants.xml'), 'w') as variantsfile:
variantsfile.write(variants_template.render())

def _sanity_check_repo(self):
"""Sanity check our repo.
- make sure we didn't compose a repo full of symlinks
- sanity check our repodata
This basically checks that pungi was run with gather_method='hardlink-or-copy' so that
we get a repository with either hardlinks or copied files.
This means that we when we go and sync generated repositories out, we do not need to take
special case to copy the target files rather than symlinks.
"""
self.log.info("Running sanity checks on %s" % self.path)

try:
arches = os.listdir(os.path.join(self.path, 'compose', 'Everything'))
except Exception:
self.log.exception('Empty compose folder? Compose thrown out')
self._toss_out_repo()
raise

if len(arches) == 0:
self.log.error('Empty compose, compose thrown out')
self._toss_out_repo()
raise Exception('Empty compose found')

for arch in arches:
# sanity check our repodata
try:
if arch == 'source':
repodata = os.path.join(self.path, 'compose',
'Everything', arch, 'tree', 'repodata')
sanity_check_repodata(repodata, source=True)
else:
repodata = os.path.join(self.path, 'compose',
'Everything', arch, 'os', 'repodata')
sanity_check_repodata(repodata, source=False)
except Exception:
self.log.exception("Repodata sanity check failed, compose thrown out")
self._toss_out_repo()
raise

# make sure that pungi didn't symlink our packages
try:
if arch == 'source':
dirs = [('tree', 'Packages')]
else:
dirs = [('debug', 'tree', 'Packages'), ('os', 'Packages')]

# Example of full path we are checking:
# self.path/compose/Everything/os/Packages/s/something.rpm
for checkdir in dirs:
checkdir = os.path.join(self.path, 'compose', 'Everything', arch, *checkdir)
subdirs = os.listdir(checkdir)
# subdirs is the self.path/compose/Everything/os/Packages/{a,b,c,...}/ dirs
#
# Let's check the first file in each subdir. If they are correct, we'll assume
# the rest is correct
# This is to avoid tons and tons of IOPS for a bunch of files put in in the
# same way
for subdir in subdirs:
for checkfile in os.listdir(os.path.join(checkdir, subdir)):
if not checkfile.endswith('.rpm'):
continue
if os.path.islink(os.path.join(checkdir, subdir, checkfile)):
self.log.error('Pungi out directory contains at least one '
'symlink at %s', checkfile)
raise Exception('Symlinks found')
# We have checked the first rpm in the subdir
break
except Exception:
self.log.exception('Unable to check pungi mashed repositories, compose thrown out')
self._toss_out_repo()
raise

return True


class ModuleComposerThread(PungiComposerThread):
"""Run Pungi with configs that produce module repositories."""
Expand Down
Loading

0 comments on commit e51f6d2

Please sign in to comment.