Skip to content
This repository
Newer
Older
100755 1902 lines (1615 sloc) 70.407 kb
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1 #!/usr/bin/python
2
3 ### This program is free software; you can redistribute it and/or modify
4 ### it under the terms of the GNU Library General Public License as published by
5 ### the Free Software Foundation; version 2 only
6 ###
7 ### This program is distributed in the hope that it will be useful,
8 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
9 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 ### GNU Library General Public License for more details.
11 ###
12 ### You should have received a copy of the GNU Library General Public License
13 ### along with this program; if not, write to the Free Software
14 ### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
b776296e »
2007-06-11 Allow the RHN/up2date code to live in /usr/share/mrepo/rhn/ (Frederic…
15 ### Copyright 2004-2007 Dag Wieers <dag@wieers.com>
b13ed519 »
2006-12-04 Changing Depo into mrepo.
16
aad5961f »
2012-05-25 Pylint/PEP8 fixes
17 from __future__ import generators # for Python 2.2
37d7975b »
2011-07-27 mrepo: importing more than one module at a time is bad coding style
18
19 import ConfigParser
20 import getopt
21 import glob
22 import os
23 import re
9fa1de46 »
2011-09-13 Add support for fetching YUM repos via reposync (closes GH-21)
24 import tempfile
33d77150 »
2011-07-27 mrepo: use hashlib instead of sha on newer Python platforms
25
26 # Python >= 2.5
27 try:
28 from hashlib import sha1 as sha1hash
29 # Python <= 2.4
30 except ImportError:
31 from sha import new as sha1hash
32
37d7975b »
2011-07-27 mrepo: importing more than one module at a time is bad coding style
33 import shutil
34 import sys
b13ed519 »
2006-12-04 Changing Depo into mrepo.
35 import time
37d7975b »
2011-07-27 mrepo: importing more than one module at a time is bad coding style
36 import types
37 import urlparse
b13ed519 »
2006-12-04 Changing Depo into mrepo.
38
39 __version__ = "$Revision$"
40 # $Source$
41
21918345 »
2010-03-20 Preparing release 0.8.7.
42 VERSION = '0.8.7'
b13ed519 »
2006-12-04 Changing Depo into mrepo.
43
44 archs = {
10288029 »
2007-05-12 Retab
45 'alpha': ('alpha', 'alphaev5', 'alphaev56', 'alphaev6', 'alphaev67'),
46 'i386': ('i386', 'i486', 'i586', 'i686', 'athlon'),
47 'ia64': ('i386', 'i686', 'ia64'),
dfe4c217 »
2007-10-19 Updates
48 'ppc': ('ppc', ),
ce9cbc5a »
2007-10-03 Fixes small groupfile fix for createrepo.
49 'ppc64': ('ppc', 'ppc64', 'ppc64pseries', 'ppc64iseries'),
b776296e »
2007-06-11 Allow the RHN/up2date code to live in /usr/share/mrepo/rhn/ (Frederic…
50 'x86_64': ('i386', 'i486', 'i586', 'i686', 'athlon', 'x86_64', 'amd64', 'ia32e'),
ce9cbc5a »
2007-10-03 Fixes small groupfile fix for createrepo.
51 'sparc64': ('sparc', 'sparcv8', 'sparcv9', 'sparc64'),
52 'sparc64v': ('sparc', 'sparcv8', 'sparcv9', 'sparcv9v', 'sparc64', 'sparc64v'),
10288029 »
2007-05-12 Retab
53 's390': ('s390', ),
dfe4c217 »
2007-10-19 Updates
54 's390x': ('s390', 's390x'),
b13ed519 »
2006-12-04 Changing Depo into mrepo.
55 }
56
57 variables = {}
58
59 enable = ('yes', 'on', 'true', '1')
60 disable = ('no', 'off', 'false', '0')
61
62 ### Register rhn and rhns as a known schemes
9fa1de46 »
2011-09-13 Add support for fetching YUM repos via reposync (closes GH-21)
63 for scheme in ('rhn', 'rhns', 'you', 'reposync', 'reposyncs'):
10288029 »
2007-05-12 Retab
64 urlparse.uses_netloc.insert(0, scheme)
65 urlparse.uses_query.insert(0, scheme)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
66
aad5961f »
2012-05-25 Pylint/PEP8 fixes
67
b13ed519 »
2006-12-04 Changing Depo into mrepo.
68 class Options:
10288029 »
2007-05-12 Retab
69 def __init__(self, args):
70 self.configfile = '/etc/mrepo.conf'
71 self.dists = []
72 self.rhnrelease = None
73 self.force = False
74 self.dryrun = False
75 self.generate = False
76 self.quiet = False
77 self.remount = False
78 self.repos = []
79 self.types = []
80 self.umount = False
81 self.update = False
82 self.verbose = 1
83
84 try:
aad5961f »
2012-05-25 Pylint/PEP8 fixes
85 opts, args = getopt.getopt(args, 'c:d:fghnqr:t:uvx',
1a062bb7 »
2012-02-08 Fixing '--repo' long argument
86 ('config=', 'dist=', 'dry-run', 'force', 'generate', 'help', 'quiet', 'repo=',
10288029 »
2007-05-12 Retab
87 'remount', 'type=', 'umount', 'unmount', 'update', 'verbose', 'version', 'extras'))
88 except getopt.error, exc:
89 print 'mrepo: %s, try mrepo -h for a list of all the options' % str(exc)
90 sys.exit(1)
91
92 for opt, arg in opts:
93 if opt in ('-c', '--config'):
94 self.configfile = os.path.abspath(arg)
95 elif opt in ('-d', '--dist'):
96 print 'mrepo: the use of -d or --dist as an option is deprecated, use the argument list'
97 self.dists = self.dists + arg.split(',')
98 elif opt in ('-f', '--force'):
99 self.force = True
100 elif opt in ('-g', '--generate'):
101 self.generate = True
102 elif opt in ('-h', '--help'):
103 self.usage()
104 print
105 self.help()
106 sys.exit(0)
107 elif opt in ('-n', '--dry-run'):
108 self.dryrun = True
109 elif opt in ('-q', '--quiet'):
110 self.quiet = True
111 elif opt in ('-r', '--repo'):
112 self.repos = self.repos + arg.split(',')
113 elif opt in ('--remount', ):
114 self.remount = True
115 elif opt in ('-t', '--type'):
116 self.types = self.types + arg.split(',')
117 elif opt in ('-u', '--update'):
118 self.update = True
119 elif opt in ('--umount', '--unmount'):
120 self.umount = True
121 elif opt in ('-v', '--verbose'):
122 self.verbose = self.verbose + 1
123 elif opt in ('--version', ):
124 self.version()
125 sys.exit(0)
126 elif opt in ('-x', '--extras'):
127 print 'mrepo: the use of -x or --extras is deprecated, use -u and -r instead'
128 self.update = True
129
130 if not self.types:
9fa1de46 »
2011-09-13 Add support for fetching YUM repos via reposync (closes GH-21)
131 self.types = ['file', 'fish', 'ftp', 'http', 'https', 'mc', 'rhn',
132 'rhns', 'rsync', 'sftp', 'mrepo', 'you', 'reposync', 'reposyncs']
10288029 »
2007-05-12 Retab
133
134 for arg in args:
135 self.dists = self.dists + arg.split(',')
136
137 if self.quiet:
138 self.verbose = 0
139
140 if self.verbose >= 3:
141 print 'Verbosity set to level %d' % (self.verbose - 1)
142 print 'Using configfile %s' % self.configfile
143
144 def version(self):
145 print 'mrepo %s' % VERSION
146 print 'Written by Dag Wieers <dag@wieers.com>'
b776296e »
2007-06-11 Allow the RHN/up2date code to live in /usr/share/mrepo/rhn/ (Frederic…
147 print 'Homepage at http://dag.wieers.com/home-made/mrepo/'
10288029 »
2007-05-12 Retab
148 print
149 print 'platform %s/%s' % (os.name, sys.platform)
150 print 'python %s' % sys.version
151 print
152 print 'build revision $Rev$'
153
154 def usage(self):
155 print 'usage: mrepo [options] dist1 [dist2-arch ..]'
156
157 def help(self):
158 print '''Set up a distribution server from ISO files
b13ed519 »
2006-12-04 Changing Depo into mrepo.
159
61777b33 »
2006-12-04 Finalize rename to mrepo
160 mrepo options:
b13ed519 »
2006-12-04 Changing Depo into mrepo.
161 -c, --config=file specify alternative configfile
162 -f, --force force repository generation
61777b33 »
2006-12-04 Finalize rename to mrepo
163 -g, --generate generate mrepo repositories
b13ed519 »
2006-12-04 Changing Depo into mrepo.
164 -n, --dry-run show what would have been done
165 -q, --quiet minimal output
166 -r, --repo=repo1,repo2 restrict action to specific repositories
167 --remount remount distribution ISOs
281872f7 »
2012-10-31 Add --version and --type to --help output
168 -t, --type=type1,type2 mirror types to use. Default: file, fish, ftp, http, https, mc, rhn, rhns, rsync, sftp, mrepo, you
b13ed519 »
2006-12-04 Changing Depo into mrepo.
169 -u, --update fetch OS updates
170 -v, --verbose increase verbosity
281872f7 »
2012-10-31 Add --version and --type to --help output
171 --version print mrepo version information
b13ed519 »
2006-12-04 Changing Depo into mrepo.
172 -vv, -vvv, -vvvv.. increase verbosity more
173 --unmount unmount distribution ISOs
174 '''
175
aad5961f »
2012-05-25 Pylint/PEP8 fixes
176
b13ed519 »
2006-12-04 Changing Depo into mrepo.
177 class Config:
10288029 »
2007-05-12 Retab
178 def __init__(self):
179 self.read(op.configfile)
180
181 self.cachedir = self.getoption('main', 'cachedir', '/var/cache/mrepo')
182 self.lockdir = self.getoption('main', 'lockdir', '/var/cache/mrepo')
183 self.confdir = self.getoption('main', 'confdir', '/etc/mrepo.conf.d')
184 self.htmldir = self.getoption('main', 'htmldir', '/usr/share/mrepo/html')
185 self.pxelinux = self.getoption('main', 'pxelinux', '/usr/lib/syslinux/pxelinux.0')
186 self.srcdir = self.getoption('main', 'srcdir', '/var/mrepo')
187 self.tftpdir = self.getoption('main', 'tftpdir', '/tftpboot/mrepo')
188 self.wwwdir = self.getoption('main', 'wwwdir', '/var/www/mrepo')
189 self.logfile = self.getoption('main', 'logfile', '/var/log/mrepo.log')
190
191 self.mailto = self.getoption('main', 'mailto', None)
192 self.mailfrom = self.getoption('main', 'mailfrom', 'mrepo@%s' % os.uname()[1])
193 self.smtpserver = self.getoption('main', 'smtp-server', 'localhost')
194
195 self.arch = self.getoption('main', 'arch', 'i386')
196 self.metadata = self.getoption('main', 'metadata', 'repomd repoview')
197
198 self.shareiso = self.getoption('main', 'shareiso', 'yes') not in disable
199
200 self.quiet = self.getoption('main', 'quiet', 'no') not in disable
201 if op.verbose == 1 and self.quiet:
202 op.verbose = 0
203
204 self.hardlink = self.getoption('main', 'hardlink', 'no') not in disable
205
3773f524 »
2008-08-15 Changes to rhnget and mrepo to reflect rhn/up2date_client changes.
206 ### FIXME: See if fuse module is loaded
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
207 self.fuseiso = self.getoption('main', 'fuseiso', 'yes') not in disable
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
208 self.unionfs = self.getoption('main', 'unionfs', 'yes') not in disable
209
10288029 »
2007-05-12 Retab
210 self.no_proxy = self.getoption('main', 'no_proxy', None)
211 self.ftp_proxy = self.getoption('main', 'ftp_proxy', None)
212 self.http_proxy = self.getoption('main', 'http_proxy', None)
213 self.https_proxy = self.getoption('main', 'https_proxy', None)
ee15ecde »
2012-02-26 add RSYNC_PROXY at mrepo
214 self.RSYNC_PROXY = self.getoption('main', 'RSYNC_PROXY', None)
10288029 »
2007-05-12 Retab
215
216 self.cmd = {}
217 self.cmd['createrepo'] = self.getoption('main', 'createrepocmd', '/usr/bin/createrepo')
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
218 self.cmd['fuseiso'] = self.getoption('main', 'fuseisocmd', '/usr/bin/fuseiso')
10288029 »
2007-05-12 Retab
219 self.cmd['genbasedir'] = self.getoption('main', 'genbasedircmd', '/usr/bin/genbasedir')
220 self.cmd['hardlink'] = self.getoption('main', 'hardlinkcmd', '/usr/sbin/hardlink')
a9ef971b »
2007-12-10 Added hardlinkpy support
221 self.cmd['hardlink++'] = self.getoption('main', 'hardlinkcppcmd', '/usr/bin/hardlink++')
222 self.cmd['hardlinkpy'] = self.getoption('main', 'hardlinkpycmd', '/usr/bin/hardlinkpy')
10288029 »
2007-05-12 Retab
223 self.cmd['lftp'] = self.getoption('main', 'lftpcmd', '/usr/bin/lftp')
224 self.cmd['mirrordir'] = self.getoption('main', 'mirrordircmd', '/usr/bin/mirrordir')
225 self.cmd['mount'] = self.getoption('main', 'mountcmd', '/bin/mount')
226 self.cmd['repoview'] = self.getoption('main', 'repoviewcmd', '/usr/bin/repoview')
9fa1de46 »
2011-09-13 Add support for fetching YUM repos via reposync (closes GH-21)
227 self.cmd['reposync'] = self.getoption('main', 'reposynccmd', '/usr/bin/reposync')
10288029 »
2007-05-12 Retab
228 self.cmd['rhnget'] = self.getoption('main', 'rhngetcmd', '/usr/bin/rhnget')
229 self.cmd['rsync'] = self.getoption('main', 'rsynccmd', '/usr/bin/rsync')
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
230 self.cmd['unionfs'] = self.getoption('main', 'unionfscmd', '/usr/bin/unionfs')
10288029 »
2007-05-12 Retab
231 self.cmd['umount'] = self.getoption('main', 'umountcmd', '/bin/umount')
dfe4c217 »
2007-10-19 Updates
232 self.cmd['youget'] = self.getoption('main', 'yougetcmd', '/usr/bin/youget')
10288029 »
2007-05-12 Retab
233 self.cmd['yumarch'] = self.getoption('main', 'yumarchcmd', '/usr/bin/yum-arch')
234
63bcb7cd »
2011-09-19 - Change default options to use --update and --database
235 self.createrepooptions = self.getoption('main', 'createrepo-options', '--pretty --database --update')
10288029 »
2007-05-12 Retab
236
237 self.lftpbwlimit = self.getoption('main', 'lftp-bandwidth-limit', None)
238 self.lftpcleanup = self.getoption('main', 'lftp-cleanup', 'yes') not in disable
239 self.lftpexcldebug = self.getoption('main', 'lftp-exclude-debug', 'yes') not in disable
240 self.lftpexclsrpm = self.getoption('main', 'lftp-exclude-srpm', 'yes') not in disable
241 self.lftpoptions = self.getoption('main', 'lftp-options', '')
242 self.lftpcommands = self.getoption('main', 'lftp-commands', '')
243 self.lftpmirroroptions = self.getoption('main', 'lftp-mirror-options', '-c -P')
244 self.lftptimeout = self.getoption('main', 'lftp-timeout', None)
245
246 self.mirrordircleanup = self.getoption('main', 'mirrordir-cleanup', 'yes') not in disable
247 self.mirrordirexcldebug = self.getoption('main', 'mirrordir-exclude-debug', 'yes') not in disable
248 self.mirrordirexclsrpm = self.getoption('main', 'mirrordir-exclude-srpm', 'yes') not in disable
249 self.mirrordiroptions = self.getoption('main', 'mirrordir-options', '')
250
9fa1de46 »
2011-09-13 Add support for fetching YUM repos via reposync (closes GH-21)
251 self.reposyncoptions = self.getoption('main', 'reposync-options', '')
252 self.reposynccleanup = self.getoption('main', 'reposync-cleanup', 'yes') not in disable
253 self.reposyncnewestonly = self.getoption('main', 'reposync-newest-only', 'no') not in disable
254
10288029 »
2007-05-12 Retab
255 self.rhnlogin = self.getoption('main', 'rhnlogin', None)
256 self.rhngetoptions = self.getoption('main', 'rhnget-options', '')
257 self.rhngetcleanup = self.getoption('main', 'rhnget-cleanup', 'yes') not in disable
258 self.rhngetdownloadall = self.getoption('main', 'rhnget-download-all', 'no') not in disable
259
260 self.rsyncbwlimit = self.getoption('main', 'rsync-bandwidth-limit', None)
261 self.rsynccleanup = self.getoption('main', 'rsync-cleanup', 'yes') not in disable
262 self.rsyncexcldebug = self.getoption('main', 'rsync-exclude-debug', 'yes') not in disable
263 self.rsyncexclsrpm = self.getoption('main', 'rsync-exclude-srpm', 'yes') not in disable
264 self.rsyncoptions = self.getoption('main', 'rsync-options', '-rtHL --partial')
265 self.rsynctimeout = self.getoption('main', 'rsync-timeout', None)
266
267 self.repoviewoptions = self.getoption('main', 'repoview-options', '')
268
269 self.alldists = []
270 self.dists = []
271
272 self.update(op.configfile)
273
274 def read(self, configfile):
275 self.cfg = ConfigParser.ConfigParser()
276
277 info(4, 'Reading config file %s' % (configfile))
278
aad5961f »
2012-05-25 Pylint/PEP8 fixes
279 (s, b, p, q, f, o) = urlparse.urlparse(configfile)
10288029 »
2007-05-12 Retab
280 if s in ('http', 'ftp', 'file'):
281 configfh = urllib.urlopen(configfile)
282 try:
283 self.cfg.readfp(configfh)
284 except ConfigParser.MissingSectionHeaderError, e:
285 die(6, 'Error accessing URL: %s' % configfile)
286 else:
287 if os.access(configfile, os.R_OK):
288 try:
289 self.cfg.read(configfile)
290 except:
291 die(7, 'Syntax error reading file: %s' % configfile)
292 else:
293 die(6, 'Error accessing file: %s' % configfile)
294
295 def update(self, configfile):
296 for section in ('variables', 'vars', 'DEFAULT'):
297 if section in self.cfg.sections():
298 for option in self.cfg.options(section):
299 variables[option] = self.cfg.get(section, option)
022ee827 »
2007-07-09 Added fuseiso support (as opposed to devloop) to allow +255 ISO and u…
300
10288029 »
2007-05-12 Retab
301 for section in self.cfg.sections():
302 if section in ('main', 'repos', 'variables', 'vars', 'DEFAULT'):
303 continue
304 else:
305 ### Check if section has appended arch
306 for arch in archs.keys():
307 if section.endswith('-%s' % arch):
aad5961f »
2012-05-25 Pylint/PEP8 fixes
308 archlist = (arch,)
10288029 »
2007-05-12 Retab
309 distname = section.split('-%s' % arch)[0]
310 break
311 else:
312 archlist = self.getoption(section, 'arch', self.arch).split()
313 distname = section
314
315 ### Add a distribution for each arch
316 for arch in archlist:
317 dist = Dist(distname, arch, self)
318 dist.arch = arch
319 dist.metadata = self.metadata.split()
320 dist.enabled = True
321 dist.promoteepoch = True
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
322 dist.fuseiso = True
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
323 dist.unionfs = True
e4f05511 »
2011-11-07 branch for systemid
324 dist.systemid = None
10288029 »
2007-05-12 Retab
325 for option in self.cfg.options(section):
326 if option in ('iso', 'name', 'release', 'repo', 'rhnrelease'):
327 setattr(dist, option, self.cfg.get(section, option))
328 elif option in ('arch', 'dist'):
329 pass
330 elif option in ('disabled',):
331 dist.enabled = self.cfg.get(section, option) in disable
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
332 elif option in ('fuseiso',):
333 dist.fuseiso = self.cfg.get(section, option) not in disable
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
334 elif option in ('unionfs',):
335 dist.unionfs = self.cfg.get(section, option) not in disable
10288029 »
2007-05-12 Retab
336 elif option in ('metadata',):
337 setattr(dist, option, self.cfg.get(section, option).split())
338 elif option in ('promoteepoch',):
339 dist.promoteepoch = self.cfg.get(section, option) not in disable
e4f05511 »
2011-11-07 branch for systemid
340 elif option in ('systemid',):
341 dist.systemid = self.cfg.get(section, option)
10288029 »
2007-05-12 Retab
342 else:
343 dist.repos.append(Repo(option, self.cfg.get(section, option), dist, self))
344
345 dist.repos.sort(reposort)
346 dist.rewrite()
347
348 self.alldists.append(dist)
349
350 if dist.enabled:
351 self.dists.append(dist)
352 else:
353 info(5, '%s: %s is disabled' % (dist.nick, dist.name))
354
355 self.alldists.sort(distsort)
356 self.dists.sort(distsort)
357
358 def getoption(self, section, option, var):
359 "Get an option from a section from configfile"
360 try:
361 var = self.cfg.get(section, option)
bed2fcc0 »
2010-11-16 Added RHEL6 product names to gensystemid
362 info(3, 'Setting option %s in section [%s] to: %s' % (option, section, var))
10288029 »
2007-05-12 Retab
363 except ConfigParser.NoSectionError, e:
364 error(5, 'Failed to find section [%s]' % section)
365 except ConfigParser.NoOptionError, e:
366 # error(4, 'Failed to find option %s in [%s], set to default: %s' % (option, section, var))
367 info(5, 'Setting option %s in section [%s] to: %s (default)' % (option, section, var))
368 return var
b13ed519 »
2006-12-04 Changing Depo into mrepo.
369
aad5961f »
2012-05-25 Pylint/PEP8 fixes
370
b13ed519 »
2006-12-04 Changing Depo into mrepo.
371 class Dist:
10288029 »
2007-05-12 Retab
372 def __init__(self, dist, arch, cf):
373 self.arch = arch
374 self.dist = dist
375 self.nick = dist + '-' + arch
376 if arch == 'none':
377 self.nick = dist
378 self.name = dist
379 self.dir = os.path.join(cf.wwwdir, self.nick)
380 self.iso = None
381 self.release = None
382 self.repos = []
383 self.rhnrelease = None
384 self.srcdir = cf.srcdir
385 self.discs = ()
386 self.isos = []
387 self.disabled = False
388
389 # def __repr__(self):
390 # for key, value in vars(self).iteritems():
391 # if isinstance(value, types.StringType):
392 # print key, '->', value
393
394 def rewrite(self):
395 "Rewrite (string) attributes to replace variables by other (string) attributes"
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
396 varlist = variables
aad5961f »
2012-05-25 Pylint/PEP8 fixes
397 varlist.update({'arch': self.arch,
398 'nick': self.nick,
399 'dist': self.dist,
400 'release': self.release,
401 'rhnrelease': self.rhnrelease})
10288029 »
2007-05-12 Retab
402 for key, value in vars(self).iteritems():
403 if isinstance(value, types.StringType):
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
404 setattr(self, key, substitute(value, varlist))
10288029 »
2007-05-12 Retab
405 for repo in self.repos:
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
406 varlist['repo'] = repo.name
407 repo.url = substitute(repo.url, varlist)
10288029 »
2007-05-12 Retab
408
409 def findisos(self):
410 "Return a list of existing ISO files"
aad5961f »
2012-05-25 Pylint/PEP8 fixes
411 if not self.iso:
412 return
10288029 »
2007-05-12 Retab
413 if not self.isos:
414 for file in self.iso.split(' '):
415 file = os.path.basename(file)
416 absfile = file
417 if not os.path.isabs(file):
418 absfile = os.path.join(cf.srcdir, self.nick, file)
419 info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
420 filelist = glob.glob(absfile)
421 if not filelist:
10288029 »
2007-05-12 Retab
422 absfile = os.path.join(cf.srcdir, self.dist, file)
423 info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
424 filelist = glob.glob(absfile)
425 if not filelist:
10288029 »
2007-05-12 Retab
426 absfile = os.path.join(cf.srcdir, 'iso', file)
427 info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
428 filelist = glob.glob(absfile)
429 if not filelist:
10288029 »
2007-05-12 Retab
430 absfile = os.path.join(cf.srcdir, file)
431 info(6, '%s: Looking for ISO files matching %s' % (self.nick, absfile))
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
432 filelist = glob.glob(absfile)
433 filelist.sort()
434 for iso in filelist:
10288029 »
2007-05-12 Retab
435 if os.path.isfile(iso) and iso not in self.isos:
436 self.isos.append(iso)
437 if self.isos:
438 info(5, '%s: Found %d ISO files at %s' % (self.nick, len(self.isos), absfile))
439 self.repos.append(Repo('os', '', self, cf))
440 self.repos.sort(reposort)
441 else:
442 info(4, '%s: No ISO files found !' % self.nick)
443
444 def listrepos(self, names=None):
445 ret = []
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
446 if names:
aad5961f »
2012-05-25 Pylint/PEP8 fixes
447 return [repo for repo in self.repos if repo.name in names]
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
448 else:
449 return self.repos
450
451 def genmetadata(self):
452 allsrcdirs = []
453 pathjoin = os.path.join
454 for repo in self.listrepos(op.repos):
455 if not repo.lock('generate'):
456 continue
457 if repo.name in ('os', 'core') and self.isos:
458 repo.url = None
aad5961f »
2012-05-25 Pylint/PEP8 fixes
459 srcdirs = [pathjoin(self.dir, disc) for disc in self.discs]
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
460 self.linksync(repo, srcdirs)
ed76a657 »
2011-07-27 mrepo: add a couple of different places to look for components
461
462 os_components = (
aad5961f »
2012-05-25 Pylint/PEP8 fixes
463 glob.glob(pathjoin(self.dir + '/disc1/*/base/comps.xml')) + # RHEL 4
464 glob.glob(pathjoin(self.dir + '/disc1/*/repodata/comps-*-core.xml')) + # RHEL 5
465 glob.glob(pathjoin(self.dir + '/disc1/repodata/*-comps*.xml')) + # RHEL 6 / CentOS 6
466 glob.glob(pathjoin(self.dir + '/disc1/repodata/comps.xml')) # Scientific Linux 6
ed76a657 »
2011-07-27 mrepo: add a couple of different places to look for components
467 )
468
469 for file in os_components:
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
470 if not os.path.exists(pathjoin(self.srcdir, self.nick, 'os-comps.xml')):
471 copy(file, pathjoin(self.srcdir, self.nick, 'os-comps.xml'))
472 allsrcdirs.extend(srcdirs)
ed76a657 »
2011-07-27 mrepo: add a couple of different places to look for components
473
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
474 else:
aad5961f »
2012-05-25 Pylint/PEP8 fixes
475 self.linksync(repo, [repo.srcdir, repo.allsrcdir])
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
476 allsrcdirs.append(repo.srcdir)
0e63df22 »
2012-02-02 Fix for missing comps for RPMS.all.
477 allsrcdirs.append(repo.allsrcdir)
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
478
479 repo.check()
480 repo.createmd()
481
482 ### After generation, write a sha1sum
483 repo.writesha1()
484 repo.unlock('generate')
485
63bcb7cd »
2011-09-19 - Change default options to use --update and --database
486 # Finally generate 'all' repsitory
aad5961f »
2012-05-25 Pylint/PEP8 fixes
487 for repo in (Repo('all', '', self, cf),):
63bcb7cd »
2011-09-19 - Change default options to use --update and --database
488 if not repo.lock('generate'):
489 continue
490
491 ### Link all srcdirs from other repositories
492 self.linksync(Repo('all', '', self, cf), allsrcdirs)
493
494 repo.check()
495 repo.createmd()
496
497 ### After generation, write a sha1sum
498 repo.writesha1()
499 repo.unlock('generate')
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
500
501 def linksync(self, repo, srcdirs=None):
502 if not srcdirs:
aad5961f »
2012-05-25 Pylint/PEP8 fixes
503 srcdirs = [repo.srcdir]
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
504 destdir = repo.wwwdir
aad5961f »
2012-05-25 Pylint/PEP8 fixes
505 srcfiles = listrpms(srcdirs, relative=destdir)
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
506 # srcfiles = [ (basename, relpath), ... ]
507 srcfiles.sort()
508 # uniq basenames
aad5961f »
2012-05-25 Pylint/PEP8 fixes
509 srcfiles = [f for i, f in enumerate(srcfiles)
510 if not i or f[0] != srcfiles[i - 1][0]]
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
511
512 info(5, '%s: Symlink %s packages from %s to %s' % (repo.dist.nick, repo.name, srcdirs, destdir))
513 mkdir(destdir)
514
515 destfiles = listrpmlinks(destdir)
516 # destfiles is a list of (link_target_base, link_target_dir) tuples
517 destfiles.sort()
518
519 pathjoin = os.path.join
520
521 def keyfunc(x):
522 # compare the basenames
523 return x[0]
524
525 changed = False
aad5961f »
2012-05-25 Pylint/PEP8 fixes
526 for srcfile, destfile in synciter(srcfiles, destfiles, key=keyfunc):
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
527 if srcfile is None:
528 # delete the link
529 base, targetdir = destfile
530 linkname = pathjoin(destdir, base)
531 info(5, 'Remove link: %s' % (linkname,))
532 if not op.dryrun:
533 os.unlink(linkname)
534 changed = True
535 elif destfile is None:
536 base, srcdir = srcfile
537 # create a new link
538 linkname = pathjoin(destdir, base)
539 target = pathjoin(srcdir, base)
540 info(5, 'New link: %s -> %s' % (linkname, target))
541 if not op.dryrun:
542 os.symlink(target, linkname)
543 changed = True
544 else:
aad5961f »
2012-05-25 Pylint/PEP8 fixes
545 # same bases
546 base, srcdir = srcfile
547 base2, curtarget = destfile
548 target = pathjoin(srcdir, base)
549 if target != curtarget:
550 info(5, 'Changed link %s: current: %s, should be: %s' % (base, curtarget, target))
551 linkname = pathjoin(destdir, base)
552 if not op.dryrun:
553 os.unlink(linkname)
554 os.symlink(target, linkname)
555 changed = True
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
556
557 if changed:
558 repo.changed = True
10288029 »
2007-05-12 Retab
559
560 def mount(self):
561 "Loopback mount all ISOs"
562 discs = []
3773f524 »
2008-08-15 Changes to rhnget and mrepo to reflect rhn/up2date_client changes.
563 mountpoints = []
10288029 »
2007-05-12 Retab
564 discnr = 0
565 if cf.shareiso:
566 mkdir(os.path.join(self.dir, 'iso'))
567 else:
568 remove(os.path.join(self.dir, 'iso'))
569 regexp = re.compile('.+[_-]CD[0-9]?\..+')
3773f524 »
2008-08-15 Changes to rhnget and mrepo to reflect rhn/up2date_client changes.
570 ### FIXME: See if fuse module is loaded
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
571 if cf.cmd['fuseiso'] and cf.fuseiso and self.fuseiso:
022ee827 »
2007-07-09 Added fuseiso support (as opposed to devloop) to allow +255 ISO and u…
572 opts = '-n'
573 extra_opts = '-oallow_other'
3773f524 »
2008-08-15 Changes to rhnget and mrepo to reflect rhn/up2date_client changes.
574 mount_cmd = cf.cmd['fuseiso']
022ee827 »
2007-07-09 Added fuseiso support (as opposed to devloop) to allow +255 ISO and u…
575 else:
576 opts = '-o loop,ro'
577 extra_opts = ''
3773f524 »
2008-08-15 Changes to rhnget and mrepo to reflect rhn/up2date_client changes.
578 mount_cmd = cf.cmd['mount']
579 if readfile('/selinux/enforce') == '1':
b838e122 »
2011-07-27 mrepo: add default level to the SELinux security context
580 opts = opts + ',context=system_u:object_r:httpd_sys_content_t:s0'
10288029 »
2007-05-12 Retab
581 for iso in self.isos:
582 if cf.shareiso:
583 symlink(iso, os.path.join(self.dir, 'iso'))
584 discnr = discnr + 1
585 discstr = 'disc'
586 if regexp.match(iso, 1):
587 discstr = 'CD'
588 disc = '%s%s' % (discstr, discnr)
589 discs.append(disc)
590 mount = os.path.join(self.dir, disc)
591 if not os.path.isfile(cf.cmd['mount']):
592 die(4, 'mount command not %s' % cf.cmd['mount'])
aad5961f »
2012-05-25 Pylint/PEP8 fixes
593 mount2 = mountpoint(iso)
f5c7ddb5 »
2007-11-07 Updates
594 if not mount2:
10288029 »
2007-05-12 Retab
595 if os.path.exists(mount) and not os.path.isdir(mount):
596 os.rename(mount, os.tempnam(os.path.dirname(mount), 'bak-'))
597 mkdir(mount)
598 if not os.path.ismount(mount):
599 info(2, '%s: Mount ISO %s to %s' % (self.nick, os.path.basename(iso), mount))
3773f524 »
2008-08-15 Changes to rhnget and mrepo to reflect rhn/up2date_client changes.
600 run('%s %s %s %s %s' % (mount_cmd, opts, iso, mount, extra_opts))
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
601 mountpoints.append(mount)
f5c7ddb5 »
2007-11-07 Updates
602 else:
603 if mount2 != mount:
604 # if os.path.exists(mount):
605 # remove(mount)
606 info(5, '%s: %s already mounted, symlink ISO to %s' % (self.nick, os.path.basename(iso), mount))
607 symlink(mount2, mount)
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
608
609 if cf.cmd['unionfs'] and cf.unionfs and self.unionfs:
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
610 ### This will be the name of our filesystem (first column of /etc/mtab)
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
611 unionfs_name = "%s-%s-fuse" % (self.dist, self.arch)
612
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
613 ### We need to make sure that our directory isn't already mounted (in the case of mrepo -g)
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
614 if not mountpoint(unionfs_name):
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
615 ### Create the 'os' directory for the merged trees.
3773f524 »
2008-08-15 Changes to rhnget and mrepo to reflect rhn/up2date_client changes.
616 unionfs_mountpoint = os.path.join(self.dir, 'os')
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
617 mkdir(unionfs_mountpoint)
aad5961f »
2012-05-25 Pylint/PEP8 fixes
618 info(2, "%s -o allow_other,fsname=%s %s %s" %
619 (cf.cmd['unionfs'], unionfs_name,
620 ':'.join(mountpoints), unionfs_mountpoint))
621 run("%s -o allow_other,fsname=%s %s %s" %
622 (cf.cmd['unionfs'], unionfs_name,
623 ':'.join(mountpoints), unionfs_mountpoint))
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
624
10288029 »
2007-05-12 Retab
625 return discs
022ee827 »
2007-07-09 Added fuseiso support (as opposed to devloop) to allow +255 ISO and u…
626
10288029 »
2007-05-12 Retab
627 def umount(self):
628 "Umount all mounted ISOs"
629 discnr = 0
630 regexp = re.compile('.+[_-]CD[0-9]?\..+')
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
631
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
632 ### Remove any unionfs mounted directories first.
633 if os.path.ismount(os.path.join(self.dir, 'os')):
634 umount_cmd = 'fusermount -u'
635 info(2, '%s %s' % (umount_cmd, os.path.join(self.dir, 'os')))
636 run('%s %s' % (umount_cmd, os.path.join(self.dir, 'os')))
45de58de »
2008-08-14 Added unionfs mountpoint and X-Mailer patch.
637
10288029 »
2007-05-12 Retab
638 for iso in self.isos:
639 discnr = discnr + 1
640 discstr = 'disc'
641 if regexp.match(iso, 1):
642 discstr = 'CD'
643 mount = os.path.join(self.dir, discstr + str(discnr))
644 if not os.path.isfile(cf.cmd['umount']):
645 die(5, 'umount command not %s' % cf.cmd['umount'])
646 if os.path.ismount(mount):
110cf353 »
2011-08-02 Revert change. On unmount, using 'iso' does not necessarily work for …
647 if mountpoint(mount):
94f74a36 »
2008-08-15 A few corrections to fuseiso and unionfs
648 info(2, '%s: Unmount ISO %s from %s' % (self.nick, os.path.basename(iso), mount))
649 run('%s %s' % (cf.cmd['umount'], mount))
650 else:
651 info(2, '%s: Unmount ISO %s from %s' % (self.nick, os.path.basename(iso), mount))
652 run('%s %s' % ('fusermount -u', mount))
10288029 »
2007-05-12 Retab
653
654 def pxe(self):
655 "Create PXE boot setup"
656 tftpbootdir = os.path.dirname(cf.tftpdir)
657 if cf.tftpdir and tftpbootdir and os.path.isdir(cf.tftpdir):
658 tftpdir = os.path.join(cf.tftpdir, self.nick)
659 mkdir(tftpdir)
660 info(1, '%s: Symlink pxe boot files to %s ' % (self.nick, tftpdir))
661 mkdir(os.path.join(tftpdir, 'pxelinux.cfg'))
662
663 ### For Red Hat
664 for file in glob.glob(self.dir + '/disc1/images/pxeboot/initrd*.img'):
665 copy(file, tftpdir)
666 for file in glob.glob(self.dir + '/disc1/images/pxeboot/vmlinuz'):
667 copy(file, tftpdir)
668
669 if cf.pxelinux:
670 copy(cf.pxelinux, tftpdir)
671
672 def html(self):
673 "Put html information in repository"
674 mkdir(self.dir)
675 if not op.dryrun:
676 open(os.path.join(self.dir, '.title'), 'w').write(self.name)
677 symlink(os.path.join(cf.htmldir, 'HEADER.repo.shtml'), os.path.join(self.dir, 'HEADER.shtml'))
678 symlink(os.path.join(cf.htmldir, 'README.repo.shtml'), os.path.join(self.dir, 'README.shtml'))
b13ed519 »
2006-12-04 Changing Depo into mrepo.
679
aad5961f »
2012-05-25 Pylint/PEP8 fixes
680
b13ed519 »
2006-12-04 Changing Depo into mrepo.
681 class Repo:
10288029 »
2007-05-12 Retab
682 def __init__(self, name, url, dist, cf):
683 self.name = name
684 self.url = url
685 self.dist = dist
686 self.srcdir = os.path.join(cf.srcdir, dist.nick, self.name)
0e63df22 »
2012-02-02 Fix for missing comps for RPMS.all.
687 self.allsrcdir = os.path.join(cf.srcdir, 'all', self.name)
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
688 self.wwwdir = os.path.join(dist.dir, 'RPMS.' + self.name)
10288029 »
2007-05-12 Retab
689
690 self.changed = False
691
88bd0620 »
2007-07-22 Small fix.
692 self.oldlist = set()
693 self.newlist = set()
b009b92f »
2007-07-22 Small fix.
694
10288029 »
2007-05-12 Retab
695 def __repr__(self):
696 # return "%s/%s" % (self.dist.nick, self.name)
697 return self.name
698
699 def mirror(self):
700 "Check URL and pass on to mirror-functions."
701 global exitcode
702
703 ### Do not mirror for repository 'all'
704 if self.name == 'all':
705 return
706
707 ### Make a snapshot of the directory
708 self.oldlist = self.rpmlist()
709 self.newlist = self.oldlist
710
711 for url in self.url.split():
712 try:
713 info(2, '%s: Mirror packages from %s to %s' % (self.dist.nick, url, self.srcdir))
714 s, l, p, q, f, o = urlparse.urlparse(url)
715 if s not in op.types:
716 info(4, 'Ignoring mirror action for type %s' % s)
717 continue
718 if s in ('rsync', ):
719 mirrorrsync(url, self.srcdir)
720 elif s in ('ftp', ):
721 if cf.cmd['mirrordir']:
722 mirrormirrordir(url, self.srcdir)
723 else:
724 mirrorlftp(url, self.srcdir)
725 elif s in ('fish', 'http', 'https', 'sftp'):
726 mirrorlftp(url, self.srcdir)
727 elif s in ('file', ''):
728 mirrorfile(url, self.srcdir)
729 elif s in ('mrepo', ):
730 mirrormrepo(url, self.srcdir)
731 elif s in ('mc', ):
732 mirrormirrordir(url, self.srcdir)
733 elif s in ('rhn', 'rhns'):
734 mirrorrhnget(url, self.srcdir, self.dist)
dfe4c217 »
2007-10-19 Updates
735 elif s in ('you', ):
736 mirroryouget(url, self.srcdir, self.dist)
9fa1de46 »
2011-09-13 Add support for fetching YUM repos via reposync (closes GH-21)
737 elif s in ('reposync', 'reposyncs'):
738 mirrorreposync(url, self.srcdir, '%s-%s' % (self.dist.nick, self.name))
10288029 »
2007-05-12 Retab
739 else:
740 error(2, 'Scheme %s:// not implemented yet (in %s)' % (s, url))
741 except mrepoMirrorException, e:
742 error(0, 'Mirroring failed for %s with message:\n %s' % (url, e.value))
743 exitcode = 2
744 if not self.url:
745 ### Create directory in case no URL is given
746 mkdir(self.srcdir)
747
748 ### Make a snapshot of the directory
749 self.newlist = self.rpmlist()
750
751 def rpmlist(self):
752 "Capture a list of packages in the repository"
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
753 filelist = set()
10288029 »
2007-05-12 Retab
754
755 ### os.walk() is a python 2.4 feature
756 # for root, dirs, files in os.walk(self.srcdir):
757 # for file in files:
758 # if os.path.exists(file) and file.endswith('.rpm'):
759 # size = os.stat(os.path.join(root, file)).st_size
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
760 # filelist.add( (file, size) )
10288029 »
2007-05-12 Retab
761
762 ### os.path.walk() goes back further
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
763 def addfile((filelist, ), path, files):
10288029 »
2007-05-12 Retab
764 for file in files:
765 if os.path.exists(os.path.join(path, file)) and file.endswith('.rpm'):
766 size = os.stat(os.path.join(path, file)).st_size
aad5961f »
2012-05-25 Pylint/PEP8 fixes
767 filelist.add((file, size))
10288029 »
2007-05-12 Retab
768
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
769 os.path.walk(self.srcdir, addfile, (filelist,))
770 return filelist
10288029 »
2007-05-12 Retab
771
772 def check(self):
773 "Return what repositories require an update and write .newsha1sum"
774 if not os.path.isdir(self.wwwdir):
775 return
776 sha1file = os.path.join(self.wwwdir, '.sha1sum')
777 remove(sha1file + '.tmp')
778 cursha1 = sha1dir(self.wwwdir)
779 if op.force:
780 pass
781 elif os.path.isfile(sha1file):
782 oldsha1 = open(sha1file).read()
783 if cursha1 != oldsha1:
784 info(2, '%s: Repository %s has new packages.' % (self.dist.nick, self.name))
785 else:
786 info(5, '%s: Repository %s has not changed. Skipping.' % (self.dist.nick, self.name))
787 return
788 else:
789 info(5, '%s: New repository %s detected.' % (self.dist.nick, self.name))
790 writesha1(sha1file + '.tmp', cursha1)
791 self.changed = True
792
793 def writesha1(self):
794 "Verify .newsha1sum and write a .sha1sum file per repository"
795 ### FIXME: Repository 'all' got lost when introducing Repo class
796 sha1file = os.path.join(self.wwwdir, '.sha1sum')
797 if os.path.isfile(sha1file + '.tmp'):
798 cursha1 = sha1dir(self.wwwdir)
799 tmpsha1 = open(sha1file + '.tmp').read()
800 remove(sha1file + '.tmp')
801 if cursha1 == tmpsha1:
802 writesha1(sha1file, cursha1)
803 else:
804 info(5, '%s: Checksum is different. expect: %s, got: %s' % (self.dist.nick, cursha1, tmpsha1))
805 info(1, '%s: Directory changed during generating %s repo, please generate again.' % (self.dist.nick, self.name))
806
807 def lock(self, action):
808 if op.dryrun:
809 return True
810 lockfile = os.path.join(cf.lockdir, self.dist.nick, action + '-' + self.name + '.lock')
811 mkdir(os.path.dirname(lockfile))
812 try:
813 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600)
814 info(6, '%s: Setting lock %s' % (self.dist.nick, lockfile))
815 os.write(fd, '%d' % os.getpid())
816 os.close(fd)
817 return True
818 except:
819 if os.path.exists(lockfile):
820 pid = open(lockfile).read()
821 if os.path.exists('/proc/%s' % pid):
822 error(0, '%s: Found existing lock %s owned by pid %s' % (self.dist.nick, lockfile, pid))
823 else:
824 info(6, '%s: Removing stale lock %s' % (self.dist.nick, lockfile))
825 os.unlink(lockfile)
826 self.lock(action)
827 return True
828 else:
829 error(0, '%s: Lockfile %s does not exist. Cannot lock. Parallel universe ?' % (self.dist.nick, lockfile))
830 return False
831
832 def unlock(self, action):
833 if op.dryrun:
834 return True
835 lockfile = os.path.join(cf.lockdir, self.dist.nick, action + '-' + self.name + '.lock')
836 info(6, '%s: Removing lock %s' % (self.dist.nick, lockfile))
837 if os.path.exists(lockfile):
838 pid = open(lockfile).read()
839 if pid == '%s' % os.getpid():
840 os.unlink(lockfile)
841 else:
842 error(0, '%s: Existing lock %s found owned by another process with pid %s. This should NOT happen.' % (self.dist.nick, lockfile, pid))
843 else:
844 error(0, '%s: Lockfile %s does not exist. Cannot unlock. Something fishy here ?' % (self.dist.nick, lockfile))
022ee827 »
2007-07-09 Added fuseiso support (as opposed to devloop) to allow +255 ISO and u…
845
10288029 »
2007-05-12 Retab
846 def createmd(self):
847 metadata = ('apt', 'createrepo', 'repomd', 'repoview', 'yum')
848 index = ('repoview',)
849
850 if not self.changed and not op.force:
851 return
852
853 try:
854 ### Generate repository metadata
855 for md in self.dist.metadata:
856 if md in ('createrepo', 'repomd'):
857 self.repomd()
858 elif md in ('yum',):
859 self.yum()
860 elif md in ('apt',):
861 self.apt()
862 elif md not in index:
863 error(0, 'The %s metadata is unknown.' % md)
864
865 ### Generate repository index
866 for md in self.dist.metadata:
867 if md in ('repoview',):
868 self.repoview()
aad5961f »
2012-05-25 Pylint/PEP8 fixes
869 elif md not in metadata:
10288029 »
2007-05-12 Retab
870 error(0, 'The %s index is unknown.' % md)
871 except mrepoGenerateException, e:
872 error(0, 'Generating repo failed for %s with message:\n %s' % (self.name, e.value))
873 exitcode = 2
874
875 def repomd(self):
876 "Create a repomd repository"
877 if not cf.cmd['createrepo']:
878 raise mrepoGenerateException('Command createrepo is not found. Skipping.')
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
879
880 ### Find the createrepo version we are using (due to groupfile usage changes)
881 createrepo_version = None
882 groupfilename = 'RPMS.%s/comps.xml' % self.name
6768c8a1 »
2011-07-27 mrepo: try to get createrepo version before genpkgmetadata
883
884 sys.path.append("/usr/share/createrepo")
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
885 try:
6768c8a1 »
2011-07-27 mrepo: try to get createrepo version before genpkgmetadata
886 try:
887 import createrepo
888 createrepo_version = createrepo.__version__
889 del createrepo
890 except ImportError:
891 import genpkgmetadata
892 createrepo_version = genpkgmetadata.__version__
893 del genpkgmetadata
894 except ImportError:
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
895 pass
6768c8a1 »
2011-07-27 mrepo: try to get createrepo version before genpkgmetadata
896 sys.path.remove("/usr/share/createrepo")
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
897
898 ### If version < 0.4.6, then use the old createrepo behaviour
899 if not createrepo_version:
900 error(0, '%s: Version of createrepo could not be found. Assuming newer than 0.4.6.' % self.dist.nick)
ce9cbc5a »
2007-10-03 Fixes small groupfile fix for createrepo.
901 elif vercmp(createrepo_version, '0.4.6') > 0:
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
902 groupfilename = 'comps.xml'
903
10288029 »
2007-05-12 Retab
904 opts = ' ' + cf.createrepooptions
905 if op.force:
9f8a3c70 »
2008-10-12 Bugfix.
906 opts = ' --pretty' + opts
10288029 »
2007-05-12 Retab
907 if op.verbose <= 2:
908 opts = ' --quiet' + opts
909 elif op.verbose >= 4:
910 opts = ' -v' + opts
911 if not self.dist.promoteepoch:
912 opts = opts + ' -n'
913 if os.path.isdir(self.wwwdir):
914 repoopts = opts
915 if cf.cachedir:
916 cachedir = os.path.join(cf.cachedir, self.dist.nick, self.name)
917 mkdir(cachedir)
918 repoopts = repoopts + ' --cachedir "%s"' % cachedir
919 if os.path.isdir(os.path.join(self.wwwdir, '.olddata')):
920 remove(os.path.join(self.wwwdir, '.olddata'))
921 groupfile = os.path.join(cf.srcdir, self.dist.nick, self.name + '-comps.xml')
922 if os.path.isfile(groupfile):
923 symlink(groupfile, os.path.join(self.wwwdir, 'comps.xml'))
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
924 repoopts = repoopts + ' --groupfile "%s"' % groupfilename
10288029 »
2007-05-12 Retab
925 info(2, '%s: Create repomd repository for %s' % (self.dist.nick, self.name))
926 ret = run('%s %s %s' % (cf.cmd['createrepo'], repoopts, self.wwwdir))
927 if ret:
928 raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['createrepo'], ret)))
929
930 def yum(self):
931 "Create a (old-style) yum repository"
932 if not cf.cmd['yumarch']:
933 return
934 opts = ''
935 if op.verbose <= 2:
936 opts = ' -q' + opts
937 elif op.verbose == 4:
938 opts = ' -v' + opts
939 elif op.verbose >= 5:
940 opts = ' -vv' + opts
941 if op.dryrun:
942 opts = opts + ' -n'
943 if os.path.exists(self.wwwdir):
944 if os.path.isdir(os.path.join(self.wwwdir, '.oldheaders')):
945 remove(os.path.join(self.wwwdir, '.oldheaders'))
946 info(2, '%s: Create (old-style) yum repository for %s' % (self.dist.nick, self.name))
947 ret = run('%s %s -l %s' % (cf.cmd['yumarch'], opts, self.wwwdir))
948 if ret:
949 raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['yumarch'], ret)))
950
951 def apt(self):
952 "Create an (old-style) apt repository"
953 if not cf.cmd['genbasedir']:
954 return
955 opts = ''
956 if op.verbose >= 3:
957 opts = ' --progress' + opts
958
959 mkdir(os.path.join(self.dist.dir, 'base'))
960
961 ### Write out /srcdir/nick/base/release
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
962 # TODO: should not be done per repository
10288029 »
2007-05-12 Retab
963 releasefile = os.path.join(self.dist.dir, 'base', 'release')
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
964 if not os.path.exists(releasefile):
10288029 »
2007-05-12 Retab
965 open(releasefile, 'w').write(
966 'Origin: %s\n'\
967 'Label: %s\n'\
968 'Suite: Unknown\n'\
969 'Codename: %s\n'\
970 'Date: unknown\n'\
971 'Architectures: %s\n'\
972 'Components: \n'\
973 'Description: %s\n'\
974 'MD5Sum:\n'\
975 % (os.uname()[1], self.dist.name, self.dist.nick, self.dist.arch, self.dist.name))
976
977 ### Write out /srcdir/nick/base/release.repo
aad5961f »
2012-05-25 Pylint/PEP8 fixes
978 releasefile = os.path.join(self.dist.dir, 'base', 'release.' + self.name)
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
979 if not os.path.exists(releasefile):
10288029 »
2007-05-12 Retab
980 open(releasefile, 'w').write(
981 'Archive: %s\n'\
982 'Component: %s\n'\
983 'Version: %s\n'\
984 'Origin: %s\n'\
985 'Label: Repository %s for %s\n'\
986 'Architecture: %s\n'\
987 'NotAutomatic: false\n'\
988 % (self.name, self.name, self.dist.release, os.uname()[1], self.name, self.dist.name, self.dist.arch))
989
990 info(2, '%s: Create (old-style) apt repository for %s' % (self.dist.nick, self.name))
991 # if self.newrepos == self.oldrepos:
992 # run('%s %s --flat --bloat --bz2only %s' % (cf.cmd['genbasedir'], opts, self.dist.dir))
993 # else:
994 ret = run('%s %s --flat --bloat --bz2only --partial %s %s' % (cf.cmd['genbasedir'], opts, self.dist.dir, self.name))
995 if ret:
996 raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['genbasedir'], ret)))
997
998 def repoview(self):
999 "Create a repoview index"
1000 if not self.changed and not op.force:
1001 return
1002 if not cf.cmd['repoview']:
1003 return
1004 opts = ''
1005 if op.force:
1006 opts = ' --force'
1007 if op.verbose <= 2:
1008 opts = ' --quiet' + opts
1009 if os.path.exists(self.wwwdir):
1010 info(2, '%s: Create Repoview index for %s' % (self.dist.nick, self.name))
1011 title = '%s repository for %s' % (self.name, self.dist.nick)
1012 ret = run('%s %s --title="%s" %s' % (cf.cmd['repoview'], opts, title, self.wwwdir))
1013 if ret:
1014 raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['repoview'], ret)))
1015 # url = 'http://mrepo/%s/RPMS.%s/' % (self.dist.nick, self.name)
1016 # ret = run('%s %s --url="%s" %s' % (cf.cmd['repoview'], opts, url, self.wwwdir))
1017 # if ret:
1018 # raise(mrepoGenerateException('%s failed with return code: %s' % (cf.cmd['repoview'], ret)))
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1019
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1020
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1021 class mySet:
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1022 def __init__(self, seq=()):
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1023 self._dict = dict([(a, True) for a in seq])
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1024
10288029 »
2007-05-12 Retab
1025 def add(self, input):
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1026 self._dict[input] = True
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1027
10288029 »
2007-05-12 Retab
1028 def difference(self, other):
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1029 return mySet([k for k in self._dict.keys() if k not in other])
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1030
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1031 def __getitem__(self, key):
1032 return self._dict[key]
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1033
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1034 def __iter__(self):
1035 return self._dict.__iter__()
1036
1037 def __repr__(self):
1038 return 'mySet(%r)' % (self._dict.keys(), )
1039
1040 __str__ = __repr__
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1041
10288029 »
2007-05-12 Retab
1042 def __len__(self):
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1043 return len(self._dict)
1044
1045 def __eq__(self, s):
1046 return self._dict == s._dict
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1047
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1048
61777b33 »
2006-12-04 Finalize rename to mrepo
1049 class mrepoMirrorException(Exception):
10288029 »
2007-05-12 Retab
1050 def __init__(self, value):
1051 self.value = value
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1052
10288029 »
2007-05-12 Retab
1053 def __str__(self):
1054 return repr(self.value)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1055
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1056
61777b33 »
2006-12-04 Finalize rename to mrepo
1057 class mrepoGenerateException(Exception):
10288029 »
2007-05-12 Retab
1058 def __init__(self, value):
1059 self.value = value
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1060
10288029 »
2007-05-12 Retab
1061 def __str__(self):
1062 return repr(self.value)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1063
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1064
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1065 def sha1dir(dir):
10288029 »
2007-05-12 Retab
1066 "Return sha1sum of a directory"
1067 files = glob.glob(dir + '/*.rpm')
1068 files.sort()
1069 output = ''
1070 for file in files:
1071 output = output + os.path.basename(file) + ' ' + str(os.stat(file).st_size) + '\n'
33d77150 »
2011-07-27 mrepo: use hashlib instead of sha on newer Python platforms
1072 return sha1hash(output).hexdigest()
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1073
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1074
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1075 def writesha1(file, sha1sum=None):
10288029 »
2007-05-12 Retab
1076 "Write out sha1sum"
1077 repodir = os.path.dirname(file)
1078 if not sha1sum:
1079 sha1sum = sha1dir(repodir)
1080 if not op.dryrun:
1081 open(file, 'w').write(sha1sum)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1082
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1083
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1084 def error(level, str):
10288029 »
2007-05-12 Retab
1085 "Output error message"
1086 if level <= op.verbose:
1087 sys.stderr.write('mrepo: %s\n' % str)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1088
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1089
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1090 def info(level, str):
10288029 »
2007-05-12 Retab
1091 "Output info message"
1092 if level <= op.verbose:
1093 sys.stdout.write('%s\n' % str)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1094
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1095
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1096 def die(ret, str):
10288029 »
2007-05-12 Retab
1097 "Print error and exit with errorcode"
1098 error(0, str)
1099 sys.exit(ret)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1100
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1101
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1102 def run(str, dryrun=False):
10288029 »
2007-05-12 Retab
1103 "Run command, accept user input, and print output when needed."
1104 str = 'exec ' + str
1105 if op.verbose <= 2:
1106 str = str + ' >/dev/null'
1107 if not op.dryrun or dryrun:
1108 info(5, 'Execute: %s' % str)
1109 # os.popen(str, 'w')
1110 return os.system(str)
1111 else:
1112 info(1, 'Not execute: %s' % str)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1113
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1114
1115 def readfile(file, len=0):
10288029 »
2007-05-12 Retab
1116 "Return content of a file"
1117 if not os.path.isfile(file):
1118 return None
1119 if len:
1120 return open(file, 'r').read(len)
1121 return open(file, 'r').read()
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1122
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1123
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1124 def writefile(file, str):
10288029 »
2007-05-12 Retab
1125 if op.dryrun:
1126 return
1127 fd = open(file, 'w')
1128 fd.write(str)
1129 fd.close()
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1130
1131 _subst_sub = re.compile('\$\{?(\w+)\}?').sub
1132
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1133
1134 def substitute(string, vars, recursion=0):
10288029 »
2007-05-12 Retab
1135 "Substitute variables from a string"
1136 if recursion > 10:
1137 raise RuntimeError, "variable substitution loop"
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1138
10288029 »
2007-05-12 Retab
1139 def _substrepl(matchobj):
1140 value = vars.get(matchobj.group(1))
1141 if value is not None:
1142 return substitute(value, vars, recursion + 1)
1143 return matchobj.group(0)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1144
10288029 »
2007-05-12 Retab
1145 string = _subst_sub(_substrepl, string)
1146 return string
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1147
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1148
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1149 def mountpoint(dev):
10288029 »
2007-05-12 Retab
1150 "Return the mountpoint of a mounted device/file"
1151 for entry in readfile('/etc/mtab').split('\n'):
1152 if entry:
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1153 cols = entry.split()
1154 if dev == cols[0]:
1155 return cols[1]
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1156
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1157
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1158 def distsort(a, b):
10288029 »
2007-05-12 Retab
1159 return cmp(a.nick, b.nick)
3773f524 »
2008-08-15 Changes to rhnget and mrepo to reflect rhn/up2date_client changes.
1160
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1161
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1162 def reposort(a, b):
10288029 »
2007-05-12 Retab
1163 return cmp(a.name, b.name)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1164
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1165
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
1166 def vercmp(a, b):
1167 al = a.split('.')
1168 bl = b.split('.')
1169 minlen = min(len(al), len(bl))
1170 for i in range(1, minlen):
1171 if cmp(al[i], bl[i]) < 0:
1172 return -1
1173 elif cmp(al[i], bl[i]) > 0:
1174 return 1
cebbca5c »
2007-05-14 Small improvements.
1175 return cmp(len(al), len(bl))
8f3ebff7 »
2007-05-14 Check whether version of createrepo is older than 0.4.6 (Francois Auc…
1176
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1177
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1178 def symlinkglob(str, *targets):
10288029 »
2007-05-12 Retab
1179 "Symlink files to multiple targets"
1180 for file in glob.glob(str):
1181 for target in targets:
1182 mkdir(target)
1183 symlink(file, target)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1184
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1185
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1186 def abspath(path, reference):
10288029 »
2007-05-12 Retab
1187 "Make absolute path from reference"
1188 return os.path.normpath(os.path.join(path, reference))
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1189
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1190
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1191 def relpath(path, reference):
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1192 """Make relative path from reference
1193 if reference is a directory, it must end with a /"""
10288029 »
2007-05-12 Retab
1194 common = os.path.commonprefix([path, reference])
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1195 common = common[0:common.rfind('/') + 1]
10288029 »
2007-05-12 Retab
1196 (uncommon, targetName) = os.path.split(reference.replace(common, '', 1))
1197 if uncommon:
1198 newpath = []
1199 for component in uncommon.split('/'):
1200 newpath.append('..')
1201 newpath.append(path.replace(common, '', 1))
1202 return '/'.join(newpath)
1203 else:
1204 return path
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1205
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1206
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1207 def symlink(src, dst):
10288029 »
2007-05-12 Retab
1208 "Create a symbolic link, force if dst exists"
1209 if op.dryrun:
1210 return
1211 elif os.path.islink(dst):
1212 if os.path.samefile(src, abspath(os.readlink(dst), src)):
1213 return
1214 os.unlink(dst)
1215 elif os.path.isdir(dst):
1216 if os.path.isdir(src):
1217 if os.path.samefile(src, dst):
1218 return
1219 else:
1220 dst = os.path.join(dst, os.path.basename(src))
1221 symlink(src, dst)
1222 return
1223 elif os.path.isfile(dst):
1224 if os.path.samefile(src, dst):
1225 return
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1226 os.rename(dst, dst + '.mrepobak')
bc37c096 »
2007-05-16 Check for directory when creating before symlinking into it (Bruno Co…
1227
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1228 ### Not using filecmp increases speed with 15%
10288029 »
2007-05-12 Retab
1229 # if os.path.isfile(dst) and filecmp.cmp(src, dst) == 0:
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1230
10288029 »
2007-05-12 Retab
1231 src = relpath(src, dst)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1232
bc37c096 »
2007-05-16 Check for directory when creating before symlinking into it (Bruno Co…
1233 if not os.path.isdir(os.path.dirname(dst)):
10288029 »
2007-05-12 Retab
1234 mkdir(os.path.dirname(dst))
bc37c096 »
2007-05-16 Check for directory when creating before symlinking into it (Bruno Co…
1235 os.symlink(src, dst)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1236
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1237
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1238 def copy(src, dst):
10288029 »
2007-05-12 Retab
1239 "Copy a file, force if dst exists"
1240 if op.dryrun:
1241 return
1242 if os.path.isdir(dst):
1243 dst = os.path.join(dst, os.path.basename(src))
1244 if os.path.islink(dst) or os.path.isfile(dst):
1245 os.unlink(dst)
1246 mkdir(os.path.dirname(dst))
1247 if not os.path.exists(dst):
1248 if os.path.isfile(src):
1249 shutil.copy2(src, dst)
1250 elif os.path.isdir(src):
1251 shutil.copytree(src, dst)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1252
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1253
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1254 def remove(file):
10288029 »
2007-05-12 Retab
1255 "Remove files or directories"
1256 if isinstance(file, types.StringType):
1257 if op.dryrun:
1258 return
1259 if os.path.islink(file):
1260 os.unlink(file)
1261 elif os.path.isdir(file):
1262 try:
1263 os.rmdir(file)
1264 except:
1265 os.path.walk(file, removedir, ())
1266 os.rmdir(file)
1267 elif os.path.isfile(file) or os.path.islink(file):
1268 os.unlink(file)
1269 else:
1270 for f in file:
1271 remove(f)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1272
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1273
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1274 def removedir(void, dir, files):
10288029 »
2007-05-12 Retab
1275 for file in files:
1276 remove(os.path.join(dir, file))
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1277
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1278
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1279 def mkdir(path):
10288029 »
2007-05-12 Retab
1280 "Create a directory, and parents if needed"
1281 if op.dryrun:
1282 return
1283 if os.path.islink(path):
1284 os.unlink(path)
1285 if not os.path.exists(path):
1286 os.makedirs(path)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1287
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1288
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1289 def mirrorrsync(url, path):
10288029 »
2007-05-12 Retab
1290 "Mirror everything from an rsync:// URL"
1291 if not cf.cmd['rsync']:
2756ecdc »
2011-09-13 mrepo: fixed a typo in the error messages
1292 error(1, 'rsync was not found. rsync support is therefore disabled.')
10288029 »
2007-05-12 Retab
1293 return
1294 mkdir(path)
1295
1296 opts = cf.rsyncoptions
1297 if op.verbose <= 2:
1298 opts = opts + ' -q'
1299 elif op.verbose == 3:
1300 opts = opts + ' -v'
1301 elif op.verbose == 4:
1302 opts = opts + ' -v --progress'
1303 elif op.verbose == 5:
1304 opts = opts + ' -vv --progress'
1305 elif op.verbose >= 6:
1306 opts = opts + ' -vvv --progress'
1307 if op.dryrun:
1308 opts = opts + ' --dry-run'
1309 if cf.rsynctimeout:
1310 opts = opts + ' --timeout=%s' % cf.rsynctimeout
1311 if cf.rsynccleanup:
1312 opts = opts + ' --delete-after --delete-excluded'
1313 if cf.rsyncbwlimit:
1314 opts = opts + ' --bwlimit=%s' % cf.rsyncbwlimit
1315 opts = opts + ' --exclude=\"/headers/\" --exclude=\"/repodata/\"'
1316 if cf.rsyncexclsrpm:
1317 opts = opts + ' --exclude=\"*.src.rpm\" --exclude=\"/SRPMS/\"'
1318 if cf.rsyncexcldebug:
1319 opts = opts + ' --exclude=\"*-debuginfo-*.rpm\" --exclude=\"/debug/\"'
1320 opts = opts + ' --include=\"*.rpm\"'
1321 if cf.rsyncexclsrpm or cf.rsyncexcldebug:
1322 opts = opts + ' --exclude=\"*.*\"'
1323
1324 ret = run('%s %s %s %s' % (cf.cmd['rsync'], opts, url, path), dryrun=True)
1325 if ret:
1326 raise(mrepoMirrorException('Failed with return code: %s' % ret))
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1327
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1328
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1329 def mirrormirrordir(url, path):
10288029 »
2007-05-12 Retab
1330 "Mirror everything from a ftp:// or mc:// URL"
1331 if not cf.cmd['mirrordir']:
2756ecdc »
2011-09-13 mrepo: fixed a typo in the error messages
1332 error(1, 'mirrordir was not found. ftp and mc support (using mirrordir) is therefore disabled.')
10288029 »
2007-05-12 Retab
1333 return
1334 mkdir(path)
1335
1336 opts = cf.mirrordiroptions
1337 if op.verbose >= 3:
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1338 opts = opts + ' -v' * (op.verbose - 3)
10288029 »
2007-05-12 Retab
1339 if op.dryrun:
1340 opts = opts + ' --dry-run'
1341 if cf.mirrordircleanup:
1342 opts = opts + ' -k'
1343
1344 # opts = opts + ' -I \"*.rpm\"'
1345 opts = opts + ' -G \"headers\" -G \"repodata\"'
1346 if cf.mirrordirexclsrpm:
1347 opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
1348 if cf.mirrordirexcldebug:
1349 opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'
1350
1351 ret = run("%s %s '%s' '%s'" % (cf.cmd['mirrordir'], opts, url, path), dryrun=True)
1352 if ret:
1353 raise(mrepoMirrorException('Failed with return code: %s' % ret))
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1354
1355
1356 def mirrorlftp(url, path):
10288029 »
2007-05-12 Retab
1357 "Mirror everything from a http://, ftp://, sftp://, fish:// URL"
1358 if not cf.cmd['lftp']:
2756ecdc »
2011-09-13 mrepo: fixed a typo in the error messages
1359 error(1, 'lftp was not found. fish, ftp, http and sftp support (using lftp) is therefore disabled.')
10288029 »
2007-05-12 Retab
1360 return
1361 mkdir(path)
1362
1363 cmds = cf.lftpcommands + ';'
1364 # cmds = 'set dns:fatal-timeout 5'
1365 if cf.lftptimeout:
1366 cmds = cmds + ' set net:timeout %s;' % cf.lftptimeout
1367 if cf.lftpbwlimit:
1368 cmds = cmds + ' set net:limit-total-rate %s:0;' % cf.lftpbwlimit
1369
1370 opts = cf.lftpoptions
1371 if op.verbose >= 6:
1372 opts = opts + ' -d'
1373
1374 mirroropts = cf.lftpmirroroptions
1375 if op.verbose >= 3:
1376 mirroropts = mirroropts + ' -v' * (op.verbose - 2)
1377 if op.dryrun:
1378 mirroropts = mirroropts + ' --dry-run'
1379 if cf.lftpcleanup:
1380 mirroropts = mirroropts + ' -e'
1381 mirroropts = mirroropts + ' -I *.rpm -X \"/headers/\" -X \"/repodata/\"'
1382 if cf.lftpexclsrpm:
1383 mirroropts = mirroropts + ' -X \"*.src.rpm\" -X \"/SRPMS/\"'
1384 if cf.lftpexcldebug:
1385 mirroropts = mirroropts + ' -X \"*-debuginfo-*.rpm\" -X \"/debug/\"'
1386
1387 ret = run('%s %s -c \'%s mirror %s %s %s\'' % (cf.cmd['lftp'], opts, cmds, mirroropts, url, path), dryrun=True)
1388 if ret:
1389 raise(mrepoMirrorException('Failed with return code: %s' % ret))
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1390
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1391
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1392 def mirrorfile(url, path):
10288029 »
2007-05-12 Retab
1393 "Mirror everything from a file:// URL by symlinking"
1394 dir = url.replace('file://', '')
1395 # while dir.endswith('/'):
1396 # dir = dir[0:-1]
1397 if os.path.isdir(dir):
1398 symlink(dir, path)
1399 # else: ### FIXME: Only if ISO file
1400 # if not os.path.isabs(file):
1401 # file = os.path.join(cf.srcdir, 'iso', file)
42bec2cb »
2007-07-22 - Replace mrepo's Set() by one based on hashes or python's if availab…
1402 # isolist = glob.glob(file)
1403 # isolist.sort()
1404 # for iso in isolist:
10288029 »
2007-05-12 Retab
1405 # if os.path.isfile(iso):
1406 # print 'Please mount %s to %s' % (iso, path)
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1407
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1408
61777b33 »
2006-12-04 Finalize rename to mrepo
1409 def mirrormrepo(url, path):
10288029 »
2007-05-12 Retab
1410 "Mirror everything from a local mrepo mirror by symlinking"
1411 pathname = url.replace('mrepo://', '')
1412 while pathname.endswith('/'):
1413 pathname = pathname[0:-1]
1414 while path.endswith('/'):
1415 path = pathname[0:-1]
1416 symlink(os.path.join(cf.srcdir, pathname), path)
1417 # basename = os.path.basename(pathname)
1418 # symlink(os.path.join(cf.srcdir, pathname), os.path.join(path, basename))
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1419
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1420
b13ed519 »
2006-12-04 Changing Depo into mrepo.
1421 def mirrorrhnget(url, path, dist):
10288029 »
2007-05-12 Retab
1422 "Mirror everything from a rhn:// or rhns:// URL"
1423 if not cf.cmd['rhnget']:
2756ecdc »
2011-09-13 mrepo: fixed a typo in the error messages
1424 error(1, 'rhnget was not found. rhn and rhns support is therefore disabled.')
10288029 »
2007-05-12 Retab
1425 return
1426 mkdir(path)
1427
1428 opts = cf.rhngetoptions
1429 if op.verbose >= 3:
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1430 opts = opts + ' -v' * (op.verbose - 3)
10288029 »
2007-05-12 Retab
1431 if op.dryrun:
1432 opts = opts + ' --dry-run'
1433 if cf.rhngetcleanup:
1434 opts = opts + ' --delete'
1435 if cf.rhngetdownloadall:
1436 opts = opts + ' --download-all'
1437
e4f05511 »
2011-11-07 branch for systemid
1438 systemidpath = dist.systemid or os.path.join(cf.srcdir, dist.nick, 'systemid')
1439
10288029 »
2007-05-12 Retab
1440 if os.path.isfile(systemidpath):
1441 opts = opts + ' --systemid="%s"' % systemidpath
1442
1443 if dist.rhnrelease:
1444 opts = opts + ' --release="%s"' % dist.rhnrelease
1445
1446 if cf.rhnlogin:
1447 rhnlogin = cf.rhnlogin.split(':')
1448 if len(rhnlogin) > 0:
1449 opts = opts + ' --username="%s"' % rhnlogin[0]
1450 if len(rhnlogin) > 1:
1451 opts = opts + ' --password="%s"' % rhnlogin[1]
1452
1453 ## opts = opts + ' -I \"*.rpm\"'
1454 # opts = opts + ' -G \"headers\" -G \"repodata\"'
1455 # if cf.mirrordirexclsrpm:
1456 # opts = opts + ' -G \"*.src.rpm\" -G \"SRPMS\"'
1457 # if cf.mirrordirexcldebug:
1458 # opts = opts + ' -G \"*-debuginfo-*.rpm\" -G \"debug\"'
1459
1460 ret = run("%s %s '%s' '%s'" % (cf.cmd['rhnget'], opts, url, path), dryrun=True)
1461 if ret:
1462 raise(mrepoMirrorException('Failed with return code: %s' % ret))
dfe4c217 »
2007-10-19 Updates
1463
aad5961f »
2012-05-25 Pylint/PEP8 fixes
1464
dfe4c217 »
2007-10-19 Updates
1465 def mirroryouget(url, path, dist):
1466 "Mirror everything from a you:// URL"
1467 if not cf.cmd['youget']:
2756ecdc »
2011-09-13 mrepo: fixed a typo in the error messages
1468 error(1, 'youget was not found. YOU support is therefore disabled.')
dfe4c217 »
2007-10-19 Updates
1469 return
1470 mkdir(path)
1471