Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
tree: 4ead9b36e0
Fetching contributors…

Cannot retrieve contributors at this time

executable file 1840 lines (1601 sloc) 69.78 kB
#!/usr/bin/python
### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU Library General Public License as published by
### the Free Software Foundation; version 2 only
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
### GNU Library General Public License for more details.
###
### You should have received a copy of the GNU Library General Public License
### along with this program; if not, write to the Free Software
### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
### Copyright 2004-2007 Dag Wieers <dag@wieers.com>
from __future__ import generators # for Python 2.2
import ConfigParser
import getopt
import glob
import os
import re
import tempfile
# Python >= 2.5
try:
from hashlib import sha1 as sha1hash
# Python <= 2.4
except ImportError:
from sha import new as sha1hash
import shutil
import sys
import time
import traceback
import types
import urlparse
__version__ = "$Revision$"
# $Source$
VERSION = '0.8.7'
archs = {
'alpha': ('alpha', 'alphaev5', 'alphaev56', 'alphaev6', 'alphaev67'),
'i386': ('i386', 'i486', 'i586', 'i686', 'athlon'),
'ia64': ('i386', 'i686', 'ia64'),
'ppc': ('ppc', ),
'ppc64': ('ppc', 'ppc64', 'ppc64pseries', 'ppc64iseries'),
'x86_64': ('i386', 'i486', 'i586', 'i686', 'athlon', 'x86_64', 'amd64', 'ia32e'),
'sparc64': ('sparc', 'sparcv8', 'sparcv9', 'sparc64'),
'sparc64v': ('sparc', 'sparcv8', 'sparcv9', 'sparcv9v', 'sparc64', 'sparc64v'),
's390': ('s390', ),
's390x': ('s390', 's390x'),
}
variables = {}
enable = ('yes', 'on', 'true', '1')
disable = ('no', 'off', 'false', '0')
### Register rhn and rhns as a known schemes
for scheme in ('rhn', 'rhns', 'you', 'reposync', 'reposyncs'):
urlparse.uses_netloc.insert(0, scheme)
urlparse.uses_query.insert(0, scheme)
class Options:
def __init__(self, args):
self.configfile = '/etc/mrepo.conf'
self.dists = []
self.rhnrelease = None
self.force = False
self.dryrun = False
self.generate = False
self.quiet = False
self.remount = False
self.repos = []
self.types = []
self.umount = False
self.update = False
self.verbose = 1
try:
opts, args = getopt.getopt (args, 'c:d:fghnqr:t:uvx',
('config=', 'dist=', 'dry-run', 'force', 'generate', 'help', 'quiet', 'repo=',
'remount', 'type=', 'umount', 'unmount', 'update', 'verbose', 'version', 'extras'))
except getopt.error, exc:
print 'mrepo: %s, try mrepo -h for a list of all the options' % str(exc)
sys.exit(1)
for opt, arg in opts:
if opt in ('-c', '--config'):
self.configfile = os.path.abspath(arg)
elif opt in ('-d', '--dist'):
print 'mrepo: the use of -d or --dist as an option is deprecated, use the argument list'
self.dists = self.dists + arg.split(',')
elif opt in ('-f', '--force'):
self.force = True
elif opt in ('-g', '--generate'):
self.generate = True
elif opt in ('-h', '--help'):
self.usage()
print
self.help()
sys.exit(0)
elif opt in ('-n', '--dry-run'):
self.dryrun = True
elif opt in ('-q', '--quiet'):
self.quiet = True
elif opt in ('-r', '--repo'):
self.repos = self.repos + arg.split(',')
elif opt in ('--remount', ):
self.remount = True
elif opt in ('-t', '--type'):
self.types = self.types + arg.split(',')
elif opt in ('-u', '--update'):
self.update = True
elif opt in ('--umount', '--unmount'):
self.umount = True
elif opt in ('-v', '--verbose'):
self.verbose = self.verbose + 1
elif opt in ('--version', ):
self.version()
sys.exit(0)
elif opt in ('-x', '--extras'):
print 'mrepo: the use of -x or --extras is deprecated, use -u and -r instead'
self.update = True
if not self.types:
self.types = ['file', 'fish', 'ftp', 'http', 'https', 'mc', 'rhn',
'rhns', 'rsync', 'sftp', 'mrepo', 'you', 'reposync', 'reposyncs']
for arg in args:
self.dists = self.dists + arg.split(',')
if self.quiet:
self.verbose = 0
if self.verbose >= 3:
print 'Verbosity set to level %d' % (self.verbose - 1)
print 'Using configfile %s' % self.configfile
def version(self):
print 'mrepo %s' % VERSION
print 'Written by Dag Wieers <dag@wieers.com>'
print 'Homepage at http://dag.wieers.com/home-made/mrepo/'
print
print 'platform %s/%s' % (os.name, sys.platform)
print 'python %s' % sys.version
print
print 'build revision $Rev$'
def usage(self):
print 'usage: mrepo [options] dist1 [dist2-arch ..]'
def help(self):
print '''Set up a distribution server from ISO files
mrepo options:
-c, --config=file specify alternative configfile
-f, --force force repository generation
-g, --generate generate mrepo repositories
-n, --dry-run show what would have been done
-q, --quiet minimal output
-r, --repo=repo1,repo2 restrict action to specific repositories
--remount remount distribution ISOs
-u, --update fetch OS updates
-v, --verbose increase verbosity
-vv, -vvv, -vvvv.. increase verbosity more
--unmount unmount distribution ISOs
'''
class Config:
def __init__(self):
self.read(op.configfile)
self.cachedir = self.getoption('main', 'cachedir', '/var/cache/mrepo')
self.lockdir = self.getoption('main', 'lockdir', '/var/cache/mrepo')
self.confdir = self.getoption('main', 'confdir', '/etc/mrepo.conf.d')
self.htmldir = self.getoption('main', 'htmldir', '/usr/share/mrepo/html')
self.pxelinux = self.getoption('main', 'pxelinux', '/usr/lib/syslinux/pxelinux.0')
self.srcdir = self.getoption('main', 'srcdir', '/var/mrepo')
self.tftpdir = self.getoption('main', 'tftpdir', '/tftpboot/mrepo')
self.wwwdir = self.getoption('main', 'wwwdir', '/var/www/mrepo')
self.logfile = self.getoption('main', 'logfile', '/var/log/mrepo.log')
self.mailto = self.getoption('main', 'mailto', None)
self.mailfrom = self.getoption('main', 'mailfrom', 'mrepo@%s' % os.uname()[1])
self.smtpserver = self.getoption('main', 'smtp-server', 'localhost')
self.arch = self.getoption('main', 'arch', 'i386')
self.metadata = self.getoption('main', 'metadata', 'repomd repoview')
self.shareiso = self.getoption('main', 'shareiso', 'yes') not in disable
self.quiet = self.getoption('main', 'quiet', 'no') not in disable
if op.verbose == 1 and self.quiet:
op.verbose = 0
self.hardlink = self.getoption('main', 'hardlink', 'no') not in disable
### FIXME: See if fuse module is loaded
self.fuseiso = self.getoption('main', 'fuseiso', 'yes') not in disable
self.unionfs = self.getoption('main', 'unionfs', 'yes') not in disable
self.no_proxy = self.getoption('main', 'no_proxy', None)
self.ftp_proxy = self.getoption('main', 'ftp_proxy', None)
self.http_proxy = self.getoption('main', 'http_proxy', None)
self.https_proxy = self.getoption('main', 'https_proxy', None)
self.cmd = {}
self.cmd['createrepo'] = self.getoption('main', 'createrepocmd', '/usr/bin/createrepo')
self.cmd['fuseiso'] = self.getoption('main', 'fuseisocmd', '/usr/bin/fuseiso')
self.cmd['genbasedir'] = self.getoption('main', 'genbasedircmd', '/usr/bin/genbasedir')
self.cmd['hardlink'] = self.getoption('main', 'hardlinkcmd', '/usr/sbin/hardlink')
self.cmd['hardlink++'] = self.getoption('main', 'hardlinkcppcmd', '/usr/bin/hardlink++')
self.cmd['hardlinkpy'] = self.getoption('main', 'hardlinkpycmd', '/usr/bin/hardlinkpy')
self.cmd['lftp'] = self.getoption('main', 'lftpcmd', '/usr/bin/lftp')
self.cmd['mirrordir'] = self.getoption('main', 'mirrordircmd', '/usr/bin/mirrordir')
self.cmd['mount'] = self.getoption('main', 'mountcmd', '/bin/mount')
self.cmd['repoview'] = self.getoption('main', 'repoviewcmd', '/usr/bin/repoview')
self.cmd['reposync'] = self.getoption('main', 'reposynccmd', '/usr/bin/reposync')
self.cmd['rhnget'] = self.getoption('main', 'rhngetcmd', '/usr/bin/rhnget')
self.cmd['rsync'] = self.getoption('main', 'rsynccmd', '/usr/bin/rsync')
self.cmd['unionfs'] = self.getoption('main', 'unionfscmd', '/usr/bin/unionfs')
self.cmd['umount'] = self.getoption('main', 'umountcmd', '/bin/umount')
self.cmd['youget'] = self.getoption('main', 'yougetcmd', '/usr/bin/youget')
self.cmd['yumarch'] = self.getoption('main', 'yumarchcmd', '/usr/bin/yum-arch')
self.createrepooptions = self.getoption('main', 'createrepo-options', '--pretty --database --update')
self.lftpbwlimit = self.getoption('main', 'lftp-bandwidth-limit', None)
self.lftpcleanup = self.getoption('main', 'lftp-cleanup', 'yes') not in disable
self.lftpexcldebug = self.getoption('main', 'lftp-exclude-debug', 'yes') not in disable
self.lftpexclsrpm = self.getoption('main', 'lftp-exclude-srpm', 'yes') not in disable
self.lftpoptions = self.getoption('main', 'lftp-options', '')
self.lftpcommands = self.getoption('main', 'lftp-commands', '')
self.lftpmirroroptions = self.getoption('main', 'lftp-mirror-options', '-c -P')
self.lftptimeout = self.getoption('main', 'lftp-timeout', None)
self.mirrordircleanup = self.getoption('main', 'mirrordir-cleanup', 'yes') not in disable
self.mirrordirexcldebug = self.getoption('main', 'mirrordir-exclude-debug', 'yes') not in disable
self.mirrordirexclsrpm = self.getoption('main', 'mirrordir-exclude-srpm', 'yes') not in disable
self.mirrordiroptions = self.getoption('main', 'mirrordir-options', '')
self.reposyncoptions = self.getoption('main', 'reposync-options', '')
self.reposynccleanup = self.getoption('main', 'reposync-cleanup', 'yes') not in disable
self.reposyncnewestonly = self.getoption('main', 'reposync-newest-only', 'no') not in disable
self.rhnlogin = self.getoption('main', 'rhnlogin', None)
self.rhngetoptions = self.getoption('main', 'rhnget-options', '')
self.rhngetcleanup = self.getoption('main', 'rhnget-cleanup', 'yes') not in disable
self.rhngetdownloadall = self.getoption('main', 'rhnget-download-all', 'no') not in disable
self.rsyncbwlimit = self.getoption('main', 'rsync-bandwidth-limit', None)
self.rsynccleanup = self.getoption('main', 'rsync-cleanup', 'yes') not in disable
self.rsyncexcldebug = self.getoption('main', 'rsync-exclude-debug', 'yes') not in disable
self.rsyncexclsrpm = self.getoption('main', 'rsync-exclude-srpm', 'yes') not in disable
self.rsyncoptions = self.getoption('main', 'rsync-options', '-rtHL --partial')
self.rsynctimeout = self.getoption('main', 'rsync-timeout', None)
self.repoviewoptions = self.getoption('main', 'repoview-options', '')
self.alldists = []
self.dists = []
self.update(op.configfile)
def read(self, configfile):
self.cfg = ConfigParser.ConfigParser()
info(4, 'Reading config file %s' % (configfile))
(s,b,p,q,f,o) = urlparse.urlparse(configfile)
if s in ('http', 'ftp', 'file'):
configfh = urllib.urlopen(configfile)
try:
self.cfg.readfp(configfh)
except ConfigParser.MissingSectionHeaderError, e:
die(6, 'Error accessing URL: %s' % configfile)
else:
if os.access(configfile, os.R_OK):
try:
self.cfg.read(configfile)
except:
die(7, 'Syntax error reading file: %s' % configfile)
else:
die(6, 'Error accessing file: %s' % configfile)
def update(self, configfile):
for section in ('variables', 'vars', 'DEFAULT'):
if section in self.cfg.sections():
for option in self.cfg.options(section):
variables[option] = self.cfg.get(section, option)
for section in self.cfg.sections():
if section in ('main', 'repos', 'variables', 'vars', 'DEFAULT'):
continue
else:
### Check if section has appended arch
for arch in archs.keys():
if section.endswith('-%s' % arch):
archlist = ( arch, )
distname = section.split('-%s' % arch)[0]
break
else:
archlist = self.getoption(section, 'arch', self.arch).split()
distname = section
### Add a distribution for each arch
for arch in archlist:
dist = Dist(distname, arch, self)
dist.arch = arch
dist.metadata = self.metadata.split()
dist.enabled = True
dist.promoteepoch = True
dist.fuseiso = True
dist.unionfs = True
dist.systemid = None
for option in self.cfg.options(section):
if option in ('iso', 'name', 'release', 'repo', 'rhnrelease'):
setattr(dist, option, self.cfg.get(section, option))
elif option in ('arch', 'dist'):
pass
elif option in ('disabled',):
dist.enabled = self.cfg.get(section, option) in disable
elif option in ('fuseiso',):
dist.fuseiso = self.cfg.get(section, option) not in disable
elif option in ('unionfs',):
dist.unionfs = self.cfg.get(section, option) not in disable
elif option in ('metadata',):
setattr(dist, option, self.cfg.get(section, option).split())
elif option in ('promoteepoch',):
dist.promoteepoch = self.cfg.get(section, option) not in disable
elif option in ('systemid',):
dist.systemid = self.cfg.get(section, option)
else:
dist.repos.append(Repo(option, self.cfg.get(section, option), dist, self))
dist.repos.sort(reposort)
dist.rewrite()
self.alldists.append(dist)
if dist.enabled:
self.dists.append(dist)
else:
info(5, '%s: %s is disabled' % (dist.nick, dist.name))
self.alldists.sort(distsort)
self.dists.sort(distsort)
def getoption(self, section, option, var):
"Get an option from a section from configfile"
try:
var = self.cfg.get(section, option)
info(3, 'Setting option %s in section [%s] to: %s' % (option, section, var))
except ConfigParser.NoSectionError, e:
error(5, 'Failed to find section [%s]' % section)
except ConfigParser.NoOptionError, e:
# error(4, 'Failed to find option %s in [%s], set to default: %s' % (option, section, var))
info(5, 'Setting option %s in section [%s] to: %s (default)' % (option, section, var))
return var
class Dist:
def __init__(self, dist, arch, cf):
self.arch = arch
self.dist = dist
self.nick = dist + '-' + arch
if arch == 'none':
self.nick = dist
self.name = dist
self.dir = os.path.join(cf.wwwdir, self.nick)
self.iso = None
self.release = None
self.repos = []
self.rhnrelease = None
self.srcdir = cf.srcdir
self.discs = ()
self.isos = []
self.disabled = False
# def __repr__(self):
# for key, value in vars(self).iteritems():
# if isinstance(value, types.StringType):
# print key, '->', value
def rewrite(self):
"Rewrite (string) attributes to replace variables by other (string) attributes"
varlist = variables
varlist.update({ 'arch': self.arch, 'nick': self.nick, 'dist': self.dist,
'release': self.release, 'rhnrelease': self.rhnrelease })
for key, value in vars(self).iteritems():
if isinstance(value, types.StringType):
setattr(self, key, substitute(value, varlist))
for repo in self.repos:
varlist['repo'] = repo.name
repo.url = substitute(repo.url, varlist)
def findisos(self):
"Return a list of existing ISO files"
if not self.iso: return
if not self.isos:
for file in self.iso.split(' '):
file = os.path.basename(file)
absfile = file
if not os.path.isabs(file):
absfile = os.path.join(cf.srcdir, self.nick, file)
info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
filelist = glob.glob(absfile)
if not filelist:
absfile = os.path.join(cf.srcdir, self.dist, file)
info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
filelist = glob.glob(absfile)
if not filelist:
absfile = os.path.join(cf.srcdir, 'iso', file)
info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
filelist = glob.glob(absfile)
if not filelist:
absfile = os.path.join(cf.srcdir, file)
info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
filelist = glob.glob(absfile)
filelist.sort()
for iso in filelist:
if os.path.isfile(iso) and iso not in self.isos:
self.isos.append(iso)
if self.isos:
info(5, '%s: Found %d ISO files at %s' % (self.nick, len(self.isos), absfile))
self.repos.append(Repo('os', '', self, cf))
self.repos.sort(reposort)
else:
info(4, '%s: No ISO files found !' % self.nick)
def listrepos(self, names=None):
ret = []
if names:
return [ repo for repo in self.repos if repo.name in names ]
else:
return self.repos
def genmetadata(self):
allsrcdirs = []
pathjoin = os.path.join
for repo in self.listrepos(op.repos):
if not repo.lock('generate'):
continue
if repo.name in ('os', 'core') and self.isos:
repo.url = None
srcdirs = [ pathjoin(self.dir, disc) for disc in self.discs ]
self.linksync(repo, srcdirs)
os_components = (
glob.glob(pathjoin(self.dir + '/disc1/*/base/comps.xml')) + # RHEL 4
glob.glob(pathjoin(self.dir + '/disc1/*/repodata/comps-*-core.xml')) + # RHEL 5
glob.glob(pathjoin(self.dir + '/disc1/repodata/*-comps*.xml')) # RHEL 6 / CentOS 6
)
for file in os_components:
if not os.path.exists(pathjoin(self.srcdir, self.nick, 'os-comps.xml')):
copy(file, pathjoin(self.srcdir, self.nick, 'os-comps.xml'))
allsrcdirs.extend(srcdirs)
else:
self.linksync(repo, [ repo.srcdir, repo.allsrcdir ])
allsrcdirs.append(repo.srcdir)
allsrcdirs.append(repo.allsrcdir)
repo.check()
repo.createmd()
### After generation, write a sha1sum
repo.writesha1()
repo.unlock('generate')
# Finally generate 'all' repsitory
for repo in ( Repo('all', '', self, cf), ):
if not repo.lock('generate'):
continue
### Link all srcdirs from other repositories
self.linksync(Repo('all', '', self, cf), allsrcdirs)
repo.check()
repo.createmd()
### After generation, write a sha1sum
repo.writesha1()
repo.unlock('generate')
def linksync(self, repo, srcdirs=None):
if not srcdirs:
srcdirs = [ repo.srcdir ]
destdir = repo.wwwdir
srcfiles = listrpms(srcdirs, relative = destdir)
# srcfiles = [ (basename, relpath), ... ]
srcfiles.sort()
# uniq basenames
srcfiles = [f for i, f in enumerate(srcfiles) if not i or f[0] != srcfiles[i-1][0]]
info(5, '%s: Symlink %s packages from %s to %s' % (repo.dist.nick, repo.name, srcdirs, destdir))
mkdir(destdir)
destfiles = listrpmlinks(destdir)
# destfiles is a list of (link_target_base, link_target_dir) tuples
destfiles.sort()
pathjoin = os.path.join
def keyfunc(x):
# compare the basenames
return x[0]
changed = False
for srcfile, destfile in synciter(srcfiles, destfiles, key = keyfunc):
if srcfile is None:
# delete the link
base, targetdir = destfile
linkname = pathjoin(destdir, base)
info(5, 'Remove link: %s' % (linkname,))
if not op.dryrun:
os.unlink(linkname)
changed = True
elif destfile is None:
base, srcdir = srcfile
# create a new link
linkname = pathjoin(destdir, base)
target = pathjoin(srcdir, base)
info(5, 'New link: %s -> %s' % (linkname, target))
if not op.dryrun:
os.symlink(target, linkname)
changed = True
else:
# same bases
base, srcdir = srcfile
base2, curtarget = destfile
target = pathjoin(srcdir, base)
if target != curtarget:
info(5, 'Changed link %s: current: %s, should be: %s' % (base, curtarget, target))
linkname = pathjoin(destdir, base)
if not op.dryrun:
os.unlink(linkname)
os.symlink(target, linkname)
changed = True
if changed:
repo.changed = True
def mount(self):
"Loopback mount all ISOs"
discs = []
mountpoints = []
discnr = 0
if cf.shareiso:
mkdir(os.path.join(self.dir, 'iso'))
else:
remove(os.path.join(self.dir, 'iso'))
regexp = re.compile('.+[_-]CD[0-9]?\..+')
### FIXME: See if fuse module is loaded
if cf.cmd['fuseiso'] and cf.fuseiso and self.fuseiso:
opts = '-n'
extra_opts = '-oallow_other'
mount_cmd = cf.cmd['fuseiso']
else:
opts = '-o loop,ro'
extra_opts = ''
mount_cmd = cf.cmd['mount']
if readfile('/selinux/enforce') == '1':
opts = opts + ',context=system_u:object_r:httpd_sys_content_t:s0'
for iso in self.isos:
if cf.shareiso:
symlink(iso, os.path.join(self.dir, 'iso'))
discnr = discnr + 1
discstr = 'disc'
if regexp.match(iso, 1):
discstr = 'CD'
disc = '%s%s' % (discstr, discnr)
discs.append(disc)
mount = os.path.join(self.dir, disc)
if not os.path.isfile(cf.cmd['mount']):
die(4, 'mount command not %s' % cf.cmd['mount'])
mount2 = mountpoint(iso)
if not mount2:
if os.path.exists(mount) and not os.path.isdir(mount):
os.rename(mount, os.tempnam(os.path.dirname(mount), 'bak-'))
mkdir(mount)
if not os.path.ismount(mount):
info(2, '%s: Mount ISO %s to %s' % (self.nick, os.path.basename(iso), mount))
run('%s %s %s %s %s' % (mount_cmd, opts, iso, mount, extra_opts))
mountpoints.append(mount)
else:
if mount2 != mount:
# if os.path.exists(mount):
# remove(mount)
info(5, '%s: %s already mounted, symlink ISO to %s' % (self.nick, os.path.basename(iso), mount))
symlink(mount2, mount)
if cf.cmd['unionfs'] and cf.unionfs and self.unionfs:
### This will be the name of our filesystem (first column of /etc/mtab)
unionfs_name = "%s-%s-fuse" % (self.dist, self.arch)
### We need to make sure that our directory isn't already mounted (in the case of mrepo -g)
if not mountpoint(unionfs_name):
### Create the 'os' directory for the merged trees.
unionfs_mountpoint = os.path.join(self.dir, 'os')
mkdir(unionfs_mountpoint)
info(2, "%s -o allow_other,fsname=%s %s %s" % (cf.cmd['unionfs'], unionfs_name, ':'.join(mountpoints), unionfs_mountpoint))
run("%s -o allow_other,fsname=%s %s %s" % (cf.cmd['unionfs'], unionfs_name, ':'.join(mountpoints), unionfs_mountpoint))
return discs
def umount(self):
"Umount all mounted ISOs"
discnr = 0
regexp = re.compile('.+[_-]CD[0-9]?\..+')
### Remove any unionfs mounted directories first.
if os.path.ismount(os.path.join(self.dir, 'os')):
umount_cmd = 'fusermount -u'
info(2, '%s %s' % (umount_cmd, os.path.join(self.dir, 'os')))
run('%s %s' % (umount_cmd, os.path.join(self.dir, 'os')))
for iso in self.isos:
discnr = discnr + 1
discstr = 'disc'
if regexp.match(iso, 1):
discstr = 'CD'
mount = os.path.join(self.dir, discstr + str(discnr))
if not os.path.isfile(cf.cmd['umount']):
die(5, 'umount command not %s' % cf.cmd['umount'])
if os.path.ismount(mount):
if mountpoint(mount):
info(2, '%s: Unmount ISO %s from %s' % (self.nick, os.path.basename(iso), mount))
run('%s %s' % (cf.cmd['umount'], mount))
else:
info(2, '%s: Unmount ISO %s from %s' % (self.nick, os.path.basename(iso), mount))
run('%s %s' % ('fusermount -u', mount))
def pxe(self):
"Create PXE boot setup"
tftpbootdir = os.path.dirname(cf.tftpdir)
if cf.tftpdir and tftpbootdir and os.path.isdir(cf.tftpdir):
tftpdir = os.path.join(cf.tftpdir, self.nick)
mkdir(tftpdir)
info(1, '%s: Symlink pxe boot files to %s ' % (self.nick, tftpdir))
mkdir(os.path.join(tftpdir, 'pxelinux.cfg'))
### For Red Hat
for file in glob.glob(self.dir + '/disc1/images/pxeboot/initrd*.img'):
copy(file, tftpdir)
for file in glob.glob(self.dir + '/disc1/images/pxeboot/vmlinuz'):
copy(file, tftpdir)
if cf.pxelinux:
copy(cf.pxelinux, tftpdir)
def html(self):
"Put html information in repository"
mkdir(self.dir)
if not op.dryrun:
open(os.path.join(self.dir, '.title'), 'w').write(self.name)
symlink(os.path.join(cf.htmldir, 'HEADER.repo.shtml'), os.path.join(self.dir, 'HEADER.shtml'))
symlink(os.path.join(cf.htmldir, 'README.repo.shtml'), os.path.join(self.dir, 'README.shtml'))
class Repo:
def __init__(self, name, url, dist, cf):
self.name = name
self.url = url
self.dist = dist
self.srcdir = os.path.join(cf.srcdir, dist.nick, self.name)
self.allsrcdir = os.path.join(cf.srcdir, 'all', self.name)
self.wwwdir = os.path.join(dist.dir, 'RPMS.' + self.name)
self.changed = False
self.oldlist = set()
self.newlist = set()
def __repr__(self):
# return "%s/%s" % (self.dist.nick, self.name)
return self.name
def mirror(self):
"Check URL and pass on to mirror-functions."
global exitcode
### Do not mirror for repository 'all'
if self.name == 'all':
return
### Make a snapshot of the directory
self.oldlist = self.rpmlist()
self.newlist = self.oldlist
for url in self.url.split():
try:
info(2, '%s: Mirror packages from %s to %s' % (self.dist.nick, url, self.srcdir))
s, l, p, q, f, o = urlparse.urlparse(url)
if s not in op.types:
info(4, 'Ignoring mirror action for type %s' % s)
continue
if s in ('rsync', ):
mirrorrsync(url, self.srcdir)
elif s in ('ftp', ):
if cf.cmd['mirrordir']:
mirrormirrordir(url, self.srcdir)
else:
mirrorlftp(url, self.srcdir)
elif s in ('fish', 'http', 'https', 'sftp'):
mirrorlftp(url, self.srcdir)
elif s in ('file', ''):
mirrorfile(url, self.srcdir)
elif s in ('mrepo', ):
mirrormrepo(url, self.srcdir)
elif s in ('mc', ):
mirrormirrordir(url, self.srcdir)
elif s in ('rhn', 'rhns'):
mirrorrhnget(url, self.srcdir, self.dist)
elif s in ('you', ):
mirroryouget(url, self.srcdir, self.dist)
elif s in ('reposync', 'reposyncs'):
mirrorreposync(url, self.srcdir, '%s-%s' % (self.dist.nick, self.name))
else:
error(2, 'Scheme %s:// not implemented yet (in %s)' % (s, url))
except mrepoMirrorException, e:
error(0, 'Mirroring failed for %s with message:\n %s' % (url, e.value))
exitcode = 2
if not self.url:
### Create directory in case no URL is given
mkdir(self.srcdir)
### Make a snapshot of the directory
self.newlist = self.rpmlist()
def rpmlist(self):
"Capture a list of packages in the repository"
filelist = set()
### os.walk() is a python 2.4 feature
# for root, dirs, files in os.walk(self.srcdir):
# for file in files:
# if os.path.exists(file) and file.endswith('.rpm'):
# size = os.stat(os.path.join(root, file)).st_size
# filelist.add( (file, size) )
### os.path.walk() goes back further
def addfile((filelist, ), path, files):
for file in files:
if os.path.exists(os.path.join(path, file)) and file.endswith('.rpm'):
size = os.stat(os.path.join(path, file)).st_size
filelist.add( (file, size) )
os.path.walk(self.srcdir, addfile, (filelist,))
return filelist
def check(self):
"Return what repositories require an update and write .newsha1sum"
if not os.path.isdir(self.wwwdir):
return
sha1file = os.path.join(self.wwwdir, '.sha1sum')
remove(sha1file + '.tmp')
cursha1 = sha1dir(self.wwwdir)
if op.force:
pass
elif os.path.isfile(sha1file):
oldsha1 = open(sha1file).read()
if cursha1 != oldsha1:
info(2, '%s: Repository %s has new packages.' % (self.dist.nick, self.name))
else:
info(5, '%s: Repository %s has not changed. Skipping.' % (self.dist.nick, self.name))
return
else:
info(5, '%s: New repository %s detected.' % (self.dist.nick, self.name))
writesha1(sha1file + '.tmp', cursha1)
self.changed = True
def writesha1(self):
"Verify .newsha1sum and write a .sha1sum file per repository"
### FIXME: Repository 'all' got lost when introducing Repo class
sha1file = os.path.join(self.wwwdir, '.sha1sum')
if os.path.isfile(sha1file + '.tmp'):
cursha1 = sha1dir(self.wwwdir)
tmpsha1 = open(sha1file + '.tmp').read()
remove(sha1file + '.tmp')
if cursha1 == tmpsha1:
writesha1(sha1file, cursha1)
else:
info(5, '%s: Checksum is different. expect: %s, got: %s' % (self.dist.nick, cursha1, tmpsha1))
info(1, '%s: Directory changed during generating %s repo, please generate again.' % (self.dist.nick, self.name))
def lock(self, action):
if op.dryrun:
return True
lockfile = os.path.join(cf.lockdir, self.dist.nick, action + '-' + self.name + '.lock')
mkdir(os.path.dirname(lockfile))
try:
fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600)
info(6, '%s: Setting lock %s' % (self.dist.nick, lockfile))
os.write(fd, '%d' % os.getpid())
os.close(fd)
return True
except:
if os.path.exists(lockfile):
pid = open(lockfile).read()
if os.path.exists('/proc/%s' % pid):
error(0, '%s: Found existing lock %s owned by pid %s' % (self.dist.nick, lockfile, pid))
else:
info(6, '%s: Removing stale lock %s' % (self.dist.nick, lockfile))
os.unlink(lockfile)
self.lock(action)
return True
else:
error(0, '%s: Lockfile %s does not exist. Cannot lock. Parallel universe ?' % (self.dist.nick, lockfile))
return False
def unlock(self, action):
if op.dryrun:
return True
lockfile = os.path.join(cf.lockdir, self.dist.nick, action + '-' + self.name + '.lock')
info(6, '%s: Removing lock %s' % (self.dist.nick, lockfile))
if os.path.exists(lockfile):
pid = open(lockfile).read()
if pid == '%s' % os.getpid():
os.unlink(lockfile)
else:
error(0, '%s: Existing lock %s found owned by another process with pid %s. This should NOT happen.' % (self.dist.nick, lockfile, pid))
else:
error(0, '%s: Lockfile %s does not exist. Cannot unlock. Something fishy here ?' % (self.dist.nick, lockfile))
def createmd(self):
metadata = ('apt', 'createrepo', 'repomd', 'repoview', 'yum')
index = ('repoview',)
if not self.changed and not op.force:
return
try:
### Generate repository metadata
for md in self.dist.metadata:
if md in ('createrepo', 'repomd'):
self.repomd()
elif md in ('yum',):
self.yum()
elif md in ('apt',):
self.apt()
elif md not in index:
error(0, 'The %s metadata is unknown.' % md)
### Generate repository index
for md in self.dist.metadata:
if md in ('repoview',):
self.repoview()
elif md not in metadata:
error(0, 'The %s index is unknown.' % md)
except mrepoGenerateException, e:
error(0, 'Generating repo failed for %s with message:\n %s' % (self.name, e.value))
exitcode = 2
def repomd(self):
"Create a repomd repository"
if not cf.cmd['createrepo']:
raise mrepoGenerateException('Command createrepo is not found. Skipping.')
### Find the createrepo version we are using (due to groupfile usage changes)
createrepo_version = None
groupfilename = 'RPMS.%s/comps.xml' % self.name
sys.path.append("/usr/share/createrepo")
try:
try:
import createrepo
createrepo_version = createrepo.__version__
del createrepo
except ImportError:
import genpkgmetadata
createrepo_version = genpkgmetadata.__version__
del genpkgmetadata
except ImportError:
pass
sys.path.remove("/usr/share/createrepo")
### If version < 0.4.6, then use the old createrepo behaviour
if not createrepo_version:
error(0, '%s: Version of createrepo could not be found. Assuming newer than 0.4.6.' % self.dist.nick)
elif vercmp(createrepo_version, '0.4.6') > 0:
groupfilename = 'comps.xml'
opts = ' ' + cf.createrepooptions
if op.force:
opts = ' --pretty' + opts
if op.verbose <= 2:
opts = ' --quiet' + opts
elif op.verbose >= 4:
opts = ' -v' + opts
if not self.dist.promoteepoch:
opts = opts + ' -n'
if os.path.isdir(self.wwwdir):
repoopts = opts
if cf.cachedir:
cachedir = os.path.join(cf.cachedir, self.dist.nick, self.name)
mkdir(cachedir)
repoopts = repoopts + ' --cachedir "%s"' % cachedir
if os.path.isdir(os.path.join(self.wwwdir, '.olddata')):
remove(os.path.join(self.wwwdir, '.olddata'))
groupfile = os.path.join(cf.srcdir, self.dist.nick, self.name + '-comps.xml')
if os.path.isfile(groupfile):
symlink(groupfile, os.path.join(self.wwwdir, 'comps.xml'))
repoopts = repoopts + ' --groupfile "%s"' % groupfilename
info(2, '%s: Create repomd repository for %s' % (self.dist.nick, self.name))
ret = run('%s %s %s' % (cf.cmd['createrepo'], repoopts, self.wwwdir))
if ret:
raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['createrepo'], ret)))
def yum(self):
"Create a (old-style) yum repository"
if not cf.cmd['yumarch']:
return
opts = ''
if op.verbose <= 2:
opts = ' -q' + opts
elif op.verbose == 4:
opts = ' -v' + opts
elif op.verbose >= 5:
opts = ' -vv' + opts
if op.dryrun:
opts = opts + ' -n'
if os.path.exists(self.wwwdir):
if os.path.isdir(os.path.join(self.wwwdir, '.oldheaders')):
remove(os.path.join(self.wwwdir, '.oldheaders'))
info(2, '%s: Create (old-style) yum repository for %s' % (self.dist.nick, self.name))
ret = run('%s %s -l %s' % (cf.cmd['yumarch'], opts, self.wwwdir))
if ret:
raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['yumarch'], ret)))
def apt(self):
"Create an (old-style) apt repository"
if not cf.cmd['genbasedir']:
return
opts = ''
if op.verbose >= 3:
opts = ' --progress' + opts
mkdir(os.path.join(self.dist.dir, 'base'))
### Write out /srcdir/nick/base/release
# TODO: should not be done per repository
releasefile = os.path.join(self.dist.dir, 'base', 'release')
if not os.path.exists(releasefile):
open(releasefile, 'w').write(
'Origin: %s\n'\
'Label: %s\n'\
'Suite: Unknown\n'\
'Codename: %s\n'\
'Date: unknown\n'\
'Architectures: %s\n'\
'Components: \n'\
'Description: %s\n'\
'MD5Sum:\n'\
% (os.uname()[1], self.dist.name, self.dist.nick, self.dist.arch, self.dist.name))
### Write out /srcdir/nick/base/release.repo
releasefile = os.path.join(self.dist.dir, 'base', 'release.'+ self.name)
if not os.path.exists(releasefile):
open(releasefile, 'w').write(
'Archive: %s\n'\
'Component: %s\n'\
'Version: %s\n'\
'Origin: %s\n'\
'Label: Repository %s for %s\n'\
'Architecture: %s\n'\
'NotAutomatic: false\n'\
% (self.name, self.name, self.dist.release, os.uname()[1], self.name, self.dist.name, self.dist.arch))
info(2, '%s: Create (old-style) apt repository for %s' % (self.dist.nick, self.name))
# if self.newrepos == self.oldrepos:
# run('%s %s --flat --bloat --bz2only %s' % (cf.cmd['genbasedir'], opts, self.dist.dir))
# else:
ret = run('%s %s --flat --bloat --bz2only --partial %s %s' % (cf.cmd['genbasedir'], opts, self.dist.dir, self.name))
if ret:
raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['genbasedir'], ret)))
def repoview(self):
"Create a repoview index"
if not self.changed and not op.force:
return
if not cf.cmd['repoview']:
return
opts = ''
if op.force:
opts = ' --force'
if op.verbose <= 2:
opts = ' --quiet' + opts
if os.path.exists(self.wwwdir):
info(2, '%s: Create Repoview index for %s' % (self.dist.nick, self.name))
title = '%s repository for %s' % (self.name, self.dist.nick)
ret = run('%s %s --title="%s" %s' % (cf.cmd['repoview'], opts, title, self.wwwdir))
if ret:
raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['repoview'], ret)))
# url = 'http://mrepo/%s/RPMS.%s/' % (self.dist.nick, self.name)
# ret = run('%s %s --url="%s" %s' % (cf.cmd['repoview'], opts, url, self.wwwdir))
# if ret:
# raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['repoview'], ret)))
class mySet:
def __init__(self, seq = ()):
self._dict = dict([(a, True) for a in seq])
def add(self, input):
self._dict[input] = True
def difference(self, other):
return mySet([k for k in self._dict.keys() if k not in other])
def __getitem__(self, key):
return self._dict[key]
def __iter__(self):
return self._dict.__iter__()
def __repr__(self):
return 'mySet(%r)' % (self._dict.keys(), )
__str__ = __repr__
def __len__(self):
return len(self._dict)
def __eq__(self, s):
return self._dict == s._dict
class mrepoMirrorException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
class mrepoGenerateException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def sha1dir(dir):
"Return sha1sum of a directory"
files = glob.glob(dir + '/*.rpm')
files.sort()
output = ''
for file in files:
output = output + os.path.basename(file) + ' ' + str(os.stat(file).st_size) + '\n'
return sha1hash(output).hexdigest()
def writesha1(file, sha1sum=None):
"Write out sha1sum"
repodir = os.path.dirname(file)
if not sha1sum:
sha1sum = sha1dir(repodir)
if not op.dryrun:
open(file, 'w').write(sha1sum)
def error(level, str):
"Output error message"
if level <= op.verbose:
sys.stderr.write('mrepo: %s\n' % str)
def info(level, str):
"Output info message"
if level <= op.verbose:
sys.stdout.write('%s\n' % str)
def die(ret, str):
"Print error and exit with errorcode"
error(0, str)
sys.exit(ret)
def run(str, dryrun=False):
"Run command, accept user input, and print output when needed."
str = 'exec ' + str
if op.verbose <= 2:
str = str + ' >/dev/null'
if not op.dryrun or dryrun:
info(5, 'Execute: %s' % str)
# os.popen(str, 'w')
return os.system(str)
else:
info(1, 'Not execute: %s' % str)
def readfile(file, len = 0):
"Return content of a file"
if not os.path.isfile(file):
return None
if len:
return open(file, 'r').read(len)
return open(file, 'r').read()
def writefile(file, str):
if op.dryrun:
return
fd = open(file, 'w')
fd.write(str)
fd.close()
_subst_sub = re.compile('\$\{?(\w+)\}?').sub
def substitute(string, vars, recursion = 0):
"Substitute variables from a string"
if recursion > 10:
raise RuntimeError, "variable substitution loop"
def _substrepl(matchobj):
value = vars.get(matchobj.group(1))
if value is not None:
return substitute(value, vars, recursion + 1)
return matchobj.group(0)
string = _subst_sub(_substrepl, string)
return string
def mountpoint(dev):
"Return the mountpoint of a mounted device/file"
for entry in readfile('/etc/mtab').split('\n'):
if entry:
cols = entry.split()
if dev == cols[0]:
return cols[1]
def distsort(a, b):
return cmp(a.nick, b.nick)
def reposort(a, b):
return cmp(a.name, b.name)
def vercmp(a, b):
al = a.split('.')
bl = b.split('.')
minlen = min(len(al), len(bl))
for i in range(1, minlen):
if cmp(al[i], bl[i]) < 0:
return -1
elif cmp(al[i], bl[i]) > 0:
return 1
return cmp(len(al), len(bl))
def symlinkglob(str, *targets):
"Symlink files to multiple targets"
for file in glob.glob(str):
for target in targets:
mkdir(target)
symlink(file, target)
def abspath(path, reference):
"Make absolute path from reference"
return os.path.normpath(os.path.join(path, reference))
def relpath(path, reference):
"""Make relative path from reference
if reference is a directory, it must end with a /"""
common = os.path.commonprefix([path, reference])
common = common[0:common.rfind('/')+1]
(uncommon, targetName) = os.path.split(reference.replace(common, '', 1))
if uncommon:
newpath = []
for component in uncommon.split('/'):
newpath.append('..')
newpath.append(path.replace(common, '', 1))
return '/'.join(newpath)
else:
return path
def symlink(src, dst):
"Create a symbolic link, force if dst exists"
if op.dryrun:
return
elif os.path.islink(dst):
if os.path.samefile(src, abspath(os.readlink(dst), src)):
return
os.unlink(dst)
elif os.path.isdir(dst):
if os.path.isdir(src):
if os.path.samefile(src, dst):
return
else:
dst = os.path.join(dst, os.path.basename(src))
symlink(src, dst)
return
elif os.path.isfile(dst):
if os.path.samefile(src, dst):
return
os.rename(dst, dst+'.mrepobak')
### Not using filecmp increases speed with 15%
# if os.path.isfile(dst) and filecmp.cmp(src, dst) == 0:
src = relpath(src, dst)
if not os.path.isdir(os.path.dirname(dst)):
mkdir(os.path.dirname(dst))
os.symlink(src, dst)
def copy(src, dst):
"Copy a file, force if dst exists"
if op.dryrun:
return
if os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(src))
if os.path.islink(dst) or os.path.isfile(dst):
os.unlink(dst)
mkdir(os.path.dirname(dst))
if not os.path.exists(dst):
if os.path.isfile(src):
shutil.copy2(src, dst)
elif os.path.isdir(src):
shutil.copytree(src, dst)
def remove(file):
"Remove files or directories"
if isinstance(file, types.StringType):
if op.dryrun:
return
if os.path.islink(file):
os.unlink(file)
elif os.path.isdir(file):
try:
os.rmdir(file)
except:
os.path.walk(file, removedir, ())
os.rmdir(file)
elif os.path.isfile(file) or os.path.islink(file):
os.unlink(file)
else:
for f in file:
remove(f)
def removedir(void, dir, files):
for file in files:
remove(os.path.join(dir, file))
def mkdir(path):
"Create a directory, and parents if needed"
if op.dryrun:
return
if os.path.islink(path):
os.unlink(path)
if not os.path.exists(path):
os.makedirs(path)
def mirrorrsync(url, path):
"Mirror everything from an rsync:// URL"
if not cf.cmd['rsync']:
error(1, 'rsync was not found. rsync support is therefore disabled.')
return
mkdir(path)
opts = cf.rsyncoptions
if op.verbose <= 2:
opts = opts + ' -q'
elif op.verbose == 3:
opts = opts + ' -v'
elif op.verbose == 4:
opts = opts + ' -v --progress'
elif op.verbose == 5:
opts = opts + ' -vv --progress'
elif op.verbose >= 6:
opts = opts + ' -vvv --progress'
if op.dryrun:
opts = opts + ' --dry-run'
if cf.rsynctimeout:
opts = opts + ' --timeout=%s' % cf.rsynctimeout
if cf.rsynccleanup:
opts = opts + ' --delete-after --delete-excluded'
if cf.rsyncbwlimit:
opts = opts + ' --bwlimit=%s' % cf.rsyncbwlimit
opts = opts + ' --exclude=\"/headers/\" --exclude=\"/repodata/\"'
if cf.rsyncexclsrpm:
opts = opts + ' --exclude=\"*.src.rpm\" --exclude=\"/SRPMS/\"'
if cf.rsyncexcldebug:
opts = opts + ' --exclude=\"*-debuginfo-*.rpm\" --exclude=\"/debug/\"'
opts = opts + ' --include=\"*.rpm\"'
if cf.rsyncexclsrpm or cf.rsyncexcldebug:
opts = opts + ' --exclude=\"*.*\"'
ret = run('%s %s %s %s' % (cf.cmd['rsync'], opts, url, path), dryrun=True)
if ret:
raise(mrepoMirrorException('Failed with return code: %s' % ret))
def mirrormirrordir(url, path):
"Mirror everything from a ftp:// or mc:// URL"
if not cf.cmd['mirrordir']:
error(1, 'mirrordir was not found. ftp and mc support (using mirrordir) is therefore disabled.')
return
mkdir(path)
opts = cf.mirrordiroptions
if op.verbose >= 3:
opts = opts + ' -v' * (op.verbose - 3)
if op.dryrun:
opts = opts + ' --dry-run'
if cf.mirrordircleanup:
opts = opts + ' -k'
# opts = opts + ' -I \"*.rpm\"'
opts = opts + ' -G \"headers\" -G \"repodata\"'
if cf.mirrordirexclsrpm:
opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
if cf.mirrordirexcldebug:
opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'
ret = run("%s %s '%s' '%s'" % (cf.cmd['mirrordir'], opts, url, path), dryrun=True)
if ret:
raise(mrepoMirrorException('Failed with return code: %s' % ret))
def mirrorlftp(url, path):
"Mirror everything from a http://, ftp://, sftp://, fish:// URL"
if not cf.cmd['lftp']:
error(1, 'lftp was not found. fish, ftp, http and sftp support (using lftp) is therefore disabled.')
return
mkdir(path)
cmds = cf.lftpcommands + ';'
# cmds = 'set dns:fatal-timeout 5'
if cf.lftptimeout:
cmds = cmds + ' set net:timeout %s;' % cf.lftptimeout
if cf.lftpbwlimit:
cmds = cmds + ' set net:limit-total-rate %s:0;' % cf.lftpbwlimit
opts = cf.lftpoptions
if op.verbose >= 6:
opts = opts + ' -d'
mirroropts = cf.lftpmirroroptions
if op.verbose >= 3:
mirroropts = mirroropts + ' -v' * (op.verbose - 2)
if op.dryrun:
mirroropts = mirroropts + ' --dry-run'
if cf.lftpcleanup:
mirroropts = mirroropts + ' -e'
mirroropts = mirroropts + ' -I *.rpm -X \"/headers/\" -X \"/repodata/\"'
if cf.lftpexclsrpm:
mirroropts = mirroropts + ' -X \"*.src.rpm\" -X \"/SRPMS/\"'
if cf.lftpexcldebug:
mirroropts = mirroropts + ' -X \"*-debuginfo-*.rpm\" -X \"/debug/\"'
ret = run('%s %s -c \'%s mirror %s %s %s\'' % (cf.cmd['lftp'], opts, cmds, mirroropts, url, path), dryrun=True)
if ret:
raise(mrepoMirrorException('Failed with return code: %s' % ret))
def mirrorfile(url, path):
"Mirror everything from a file:// URL by symlinking"
dir = url.replace('file://', '')
# while dir.endswith('/'):
# dir = dir[0:-1]
if os.path.isdir(dir):
symlink(dir, path)
# else: ### FIXME: Only if ISO file
# if not os.path.isabs(file):
# file = os.path.join(cf.srcdir, 'iso', file)
# isolist = glob.glob(file)
# isolist.sort()
# for iso in isolist:
# if os.path.isfile(iso):
# print 'Please mount %s to %s' % (iso, path)
def mirrormrepo(url, path):
"Mirror everything from a local mrepo mirror by symlinking"
pathname = url.replace('mrepo://', '')
while pathname.endswith('/'):
pathname = pathname[0:-1]
while path.endswith('/'):
path = pathname[0:-1]
symlink(os.path.join(cf.srcdir, pathname), path)
# basename = os.path.basename(pathname)
# symlink(os.path.join(cf.srcdir, pathname), os.path.join(path, basename))
def mirrorrhnget(url, path, dist):
"Mirror everything from a rhn:// or rhns:// URL"
if not cf.cmd['rhnget']:
error(1, 'rhnget was not found. rhn and rhns support is therefore disabled.')
return
mkdir(path)
opts = cf.rhngetoptions
if op.verbose >= 3:
opts = opts + ' -v' * (op.verbose - 3)
if op.dryrun:
opts = opts + ' --dry-run'
if cf.rhngetcleanup:
opts = opts + ' --delete'
if cf.rhngetdownloadall:
opts = opts + ' --download-all'
systemidpath = dist.systemid or os.path.join(cf.srcdir, dist.nick, 'systemid')
if os.path.isfile(systemidpath):
opts = opts + ' --systemid="%s"' % systemidpath
if dist.rhnrelease:
opts = opts + ' --release="%s"' % dist.rhnrelease
if cf.rhnlogin:
rhnlogin = cf.rhnlogin.split(':')
if len(rhnlogin) > 0:
opts = opts + ' --username="%s"' % rhnlogin[0]
if len(rhnlogin) > 1:
opts = opts + ' --password="%s"' % rhnlogin[1]
## opts = opts + ' -I \"*.rpm\"'
# opts = opts + ' -G \"headers\" -G \"repodata\"'
# if cf.mirrordirexclsrpm:
# opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
# if cf.mirrordirexcldebug:
# opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'
ret = run("%s %s '%s' '%s'" % (cf.cmd['rhnget'], opts, url, path), dryrun=True)
if ret:
raise(mrepoMirrorException('Failed with return code: %s' % ret))
def mirroryouget(url, path, dist):
"Mirror everything from a you:// URL"
if not cf.cmd['youget']:
error(1, 'youget was not found. YOU support is therefore disabled.')
return
mkdir(path)
url = url.replace('you://', 'https://')
opts = cf.rhngetoptions
if op.verbose >= 3:
opts = opts + ' -v' * (op.verbose - 3)
if op.dryrun:
opts = opts + ' --dry-run'
if cf.rhngetcleanup:
opts = opts + ' --delete'
if cf.rhngetdownloadall:
opts = opts + ' --download-all'
credpath = os.path.join(cf.srcdir, dist.nick)
if os.path.isdir(credpath):
opts = opts + ' --credpath="%s"' % credpath
# if dist.rhnrelease:
# opts = opts + ' --release="%s"' % dist.rhnrelease
# if cf.rhnlogin:
# rhnlogin = cf.rhnlogin.split(':')
# if len(rhnlogin) > 0:
# opts = opts + ' --username="%s"' % rhnlogin[0]
# if len(rhnlogin) > 1:
# opts = opts + ' --password="%s"' % rhnlogin[1]
## opts = opts + ' -I \"*.rpm\"'
# opts = opts + ' -G \"headers\" -G \"repodata\"'
# if cf.mirrordirexclsrpm:
# opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
# if cf.mirrordirexcldebug:
# opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'
ret = run("%s %s '%s' '%s'" % (cf.cmd['youget'], opts, url, path), dryrun=True)
if ret:
raise(mrepoMirrorException('Failed with return code: %s' % ret))
def mirrorreposync(url, path, reponame):
"Mirror everything from a reposync:// URL"
if not cf.cmd['reposync']:
error(1, 'reposync was not found. reposync support is therefore disabled.')
return
mkdir(path)
url = url.replace('reposyncs://', 'https://')
url = url.replace('reposync://', 'http://')
opts = cf.reposyncoptions
if op.verbose < 3:
opts = opts + ' -q'
if op.dryrun:
opts = opts + ' --urls'
if cf.reposynccleanup:
opts = opts + ' --delete'
if cf.reposyncnewestonly:
opts = opts + ' --newest-only'
# store a temporary YUM config to use with reposync
reposync_conf_contents = """[%s]
name=%s
baseurl=%s
enabled=1
""" % (reponame, reponame, url)
(fd, reposync_conf_file) = tempfile.mkstemp(text=True)
handle = os.fdopen(fd, 'w')
handle.writelines(reposync_conf_contents)
handle.close()
ret = run("%s %s -t -c '%s' -r %s -p '%s'" % \
(cf.cmd['reposync'], opts, reposync_conf_file, reponame, path))
if ret:
raise(mrepoMirrorException('Failed with return code: %s' % ret))
# remove the temporary config
os.remove(reposync_conf_file)
def hardlink(srcdir):
info(1, 'Hardlinking duplicate packages in %s.' % srcdir)
opts = ''
if cf.cmd['hardlinkpy']:
if op.verbose <= 2:
opts = ' -v 0'
else:
opts = ' -v %d' % (op.verbose - 2)
run('%s %s %s' % (cf.cmd['hardlinkpy'], os.path.join(srcdir, ''), opts))
elif cf.cmd['hardlink++']:
if op.verbose <= 2:
opts = '>/dev/null'
run('%s %s %s' % (cf.cmd['hardlink++'], os.path.join(srcdir, ''), opts))
elif cf.cmd['hardlink']:
if op.verbose:
opts = opts + ' -' + ('v' * (op.verbose - 2))
if op.dryrun:
opts = opts + ' -n'
run('%s -c %s %s' % (cf.cmd['hardlink'], opts, os.path.join(srcdir, '')), dryrun=True)
else:
info(1, 'hardlink was not found, hardlink support is therefore disabled.')
return
def rpmlink((dist, repo), dirpath, filelist):
archlist = ['noarch', ]
if archs.has_key(dist.arch):
archlist.extend(archs[dist.arch])
else:
archlist.append(dist.arch)
for arch in archlist:
regexp = re.compile('.+[\._-]' + arch + '\.rpm$')
for file in filelist:
src = os.path.join(dirpath, file)
if os.path.islink(src) and os.path.isdir(src):
os.path.walk(src, rpmlink, (dist, repo))
elif regexp.match(file, 1):
symlink(src, os.path.join(dist.dir, 'RPMS.' + repo))
symlink(src, os.path.join(dist.dir, 'RPMS.all'))
def which(cmd):
"Find executables in PATH environment"
for path in os.environ.get('PATH','$PATH').split(':'):
if os.path.isfile(os.path.join(path, cmd)):
info(5, 'Found command %s in path %s' % (cmd, path))
return os.path.join(path, cmd)
return ''
def htmlindex():
symlink(cf.htmldir + '/HEADER.index.shtml', cf.wwwdir + '/HEADER.shtml')
symlink(cf.htmldir + '/README.index.shtml', cf.wwwdir + '/README.shtml')
def mail(subject, msg):
info(2, 'Sending mail to: %s' % cf.mailto)
try:
import smtplib
smtp = smtplib.SMTP(cf.smtpserver)
# server.set_debuglevel(1)
msg = 'Subject: [mrepo] %s\nX-Mailer: mrepo %s\n\n%s' % (subject, VERSION, msg)
for email in cf.mailto.split():
smtp.sendmail(cf.mailfrom, email, 'To: %s\n%s' % (email, msg))
smtp.quit()
except:
info(1, 'Sending mail via %s failed.' % cf.smtpserver)
def readconfig():
cf = Config()
if cf.confdir and os.path.isdir(cf.confdir):
files = glob.glob(os.path.join(cf.confdir, '*.conf'))
files.sort()
for configfile in files:
cf.read(configfile)
cf.update(configfile)
return cf
def _nextNone(iterator):
try:
return iterator.next()
except StopIteration:
return None
def synciter(a, b, key = None, keya = None, keyb = None):
"""returns an iterator that compares two ordered iterables a and b.
If keya or keyb are specified, they are called with elements of the corresponding
iterable. They should return a value that is used to compare two elements.
If keya or keyb are not specified, they default to key or to the element itself,
if key is None."""
if key is None:
key = lambda x: x
if keya is None:
keya = key
if keyb is None:
keyb = key
ai = iter(a)
bi = iter(b)
aelem = _nextNone(ai)
belem = _nextNone(bi)
while not ((aelem is None) or (belem is None)):
akey = keya(aelem)
bkey = keyb(belem)
if akey == bkey:
yield aelem, belem
aelem = _nextNone(ai)
belem = _nextNone(bi)
elif akey > bkey:
# belem missing in a
yield None, belem
belem = _nextNone(bi)
elif bkey > akey:
# aelem missing in b
yield aelem, None
aelem = _nextNone(ai)
# rest
while aelem is not None:
akey = key(aelem)
yield aelem, None
aelem = _nextNone(ai)
while belem is not None:
bkey = key(belem)
yield None, belem
belem = _nextNone(bi)
def listrpms(dirs, relative = ''):
"""return a list of rpms in the given directories as a list of (name, path) tuples
if relative is specified, return the paths relative to this directory"""
if not isinstance(dirs, (list, tuple)):
dirs = ( dirs, )
if relative and not relative.endswith('/'):
relative += '/'
isdir = os.path.isdir
pathjoin = os.path.join
pathexists = os.path.exists
def processdir(rpms, path, files):
if relative:
path2 = relpath(path, relative)
else:
path2 = path
for f in files:
pf = pathjoin(path, f)
if f.endswith('.rpm') and pathexists(pf) and not isdir(pf):
rpms.append((f, path2))
rpms = []
for dir in dirs:
if not dir.startswith('/'):
dir = pathjoin(relative, dir)
os.path.walk(dir, processdir, rpms)
rpms.sort()
return rpms
def listrpmlinks(dir):
islink = os.path.islink
readlink = os.readlink
pathjoin = os.path.join
links = []
for f in os.listdir(dir):
path = pathjoin(dir, f)
if islink(path) and f.endswith('.rpm'):
links.append((f, readlink(path)))
return links
def main():
### Check availability of commands
for cmd in cf.cmd.keys():
if not cf.cmd[cmd]:
continue
cmdlist = cf.cmd[cmd].split()
if not os.path.isfile(cmdlist[0]):
cmdlist[0] = which(cmdlist[0])
if cmdlist[0] and not os.path.isfile(cmdlist[0]):
error(4, '%s command not found as %s, support disabled' % (cmd, cmdlist[0]))
cf.cmd[cmd] = ''
else:
cf.cmd[cmd] = ' '.join(cmdlist)
if not cf.cmd['createrepo'] and not cf.cmd['yumarch'] and not cf.cmd['genbasedir']:
error(1, 'No tools found to generate repository metadata. Please install apt, yum or createrepo.')
### Set proxy-related environment variables
if cf.no_proxy:
os.environ['no_proxy'] = cf.no_proxy
if cf.ftp_proxy:
os.environ['ftp_proxy'] = cf.ftp_proxy
if cf.http_proxy:
os.environ['http_proxy'] = cf.http_proxy
if cf.https_proxy:
os.environ['https_proxy'] = cf.https_proxy
### Select list of distributions in order of appearance
if not op.dists:
dists = cf.dists
else:
dists = []
for name in op.dists:
append = False
for dist in cf.alldists:
if name == dist.nick or name == dist.dist:
dists.append(dist)
append = True
if not append:
error(1, 'Distribution %s not defined' % name)
sumnew = 0
sumremoved = 0
msg = 'The following changes to mrepo\'s repositories on %s have been made:' % os.uname()[1]
### Mounting and mirroring available distributions/repositories
for dist in dists:
dist.findisos()
### Mount ISOs
if dist.isos:
if op.umount or op.remount:
dist.umount()
if not op.umount or op.remount:
dist.discs = dist.mount()
if op.update:
msg = msg + '\n\nDist: %s (%s)' % (dist.name, dist.nick)
info(1, '%s: Updating %s' % (dist.nick, dist.name))
distnew = 0
distremoved = 0
### Downloading things
for repo in dist.listrepos(op.repos):
if not repo.lock('update'):
continue
if repo.name in ('os', 'core'):
if not dist.isos:
repo.mirror()
elif repo in dist.listrepos():
repo.mirror()
else:
info(2, '%s: Repository %s does not exist' % (dist.nick, repo.name))
repo.unlock('update')
continue
repo.unlock('update')
### files whose size has changed are in new and removed!
new = repo.newlist.difference(repo.oldlist)
removed = repo.oldlist.difference(repo.newlist)
if new or removed:
msg = msg + '\n\n\tRepo: %s' % repo.name
info(2, '%s: Repository %s changed (new: %d, removed: %d)' % (dist.nick, repo.name, len(new), len(removed)))
fd = open(cf.logfile, 'a+')
date = time.strftime("%b %d %H:%M:%S", time.gmtime())
def sortedlist(pkgs):
l = list(pkgs)
l.sort()
return l
def formatlist(pkglist):
return '\n\t' + '\n\t'.join([elem[0] for elem in pkglist])
if new:
pkglist = sortedlist(new)
info(4, '%s: New packages: %s' % (dist.nick, formatlist(pkglist)))
distnew += len(pkglist)
for element in pkglist:
fd.write('%s %s/%s Added %s (%d kiB)\n' % (date, dist.nick, repo.name, element[0], element[1]/1024))
msg = msg + '\n\t\t+ %s (%d kiB)' % (element[0], element[1]/1024)
if removed:
pkglist = sortedlist(removed)
info(4, '%s: Removed packages: %s' % (dist.nick, formatlist(pkglist)))
distremoved += len(pkglist)
for element in pkglist:
fd.write('%s %s/%s Removed %s (%d kiB)\n' % (date, dist.nick, repo.name, element[0], element[1]/1024))
msg = msg + '\n\t\t- %s (%d kiB)' % (element[0], element[1]/1024)
fd.close()
repo.changed = True
if distnew or distremoved:
msg = msg + '\n'
info(1, '%s: Distribution updated (new: %d, removed: %d)' % (dist.nick, distnew, distremoved))
sumnew = sumnew + distnew
sumremoved = sumremoved + distremoved
if sumnew or sumremoved:
subject = 'changes to %s (new: %d, removed: %d)' % (os.uname()[1], sumnew, sumremoved)
mail(subject, msg)
if not op.generate:
sys.exit(0)
htmlindex()
### Generating metadata for available distributions/repositories
for dist in dists:
dist.html()
info(1, '%s: Generating %s meta-data' % (dist.nick, dist.name))
dist.genmetadata()
dist.pxe()
if cf.hardlink and not op.dists:
hardlink(cf.srcdir)
### Unbuffered sys.stdout
sys.stdout = os.fdopen(1, 'w', 0)
sys.stderr = os.fdopen(2, 'w', 0)
### Workaround for python <= 2.2.1
try:
True, False
except NameError:
True, False = (0==0, 0!=0)
try:
set()
except NameError:
set = mySet
try:
enumerate
except NameError:
enumerate = lambda seq: zip(xrange(len(seq)), seq)
### Main entrance
if __name__ == '__main__':
exitcode = 0
op = Options(sys.argv[1:])
cf = readconfig()
try:
main()
except KeyboardInterrupt, e:
die(6, 'Exiting on user request')
# except OSError, e:
# print e.errno
# die(7, 'OSError: %s' % e)
sys.exit(exitcode)
# vim:ts=4:sw=4:et
Jump to Line
Something went wrong with that request. Please try again.