Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
branch: master
Fetching contributors…

Cannot retrieve contributors at this time

executable file 510 lines (416 sloc) 16.347 kb
#!/usr/bin/python
### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU Library General Public License as published by
### the Free Software Foundation; version 2 only
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
### GNU Library General Public License for more details.
###
### You should have received a copy of the GNU Library General Public License
### along with this program; if not, write to the Free Software
### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
### Copyright 2004-2006 Dag Wieers <dag@wieers.com>
### For SLES10 implementation, see:
### http://lists.suse.com/archive/suse-sles-e/2006-Aug/0161.html
import fnmatch
import getopt
import getpass
import gzip
import os
import sys
import types
import urllib2
import urlparse
### Python 2.5 and higher
try:
import xml.etree.ElementTree as ElementTree
except:
### Bummer, but cElementTree is considerably faster than ElementTree
try:
import cElementTree as ElementTree
except:
### Then, hopefully this works ?
try:
import ElementTree
except:
### No, it did not
print >> sys.stderr, 'Error loading python module ElementTree, please install.'
sys.exit(1)
__version__ = "$Revision: 4786 $"
# $Source$
VERSION = '0.8.4svn'
class Options:
def __init__(self, args):
self.cleanup = False
self.credpath = None
self.downloadall = False
self.dryrun = False
self.filter = None
self.list = None
self.quiet = False
self.password = None
self.username = None
self.style = 'sles10'
self.source = False
self.verbose = 1
try:
opts, args = getopt.getopt(args,
'd:hlnqp:s:u:v',
('credpath=', 'delete', 'download-all',
'dryrun', 'filter=', 'help', 'list',
'password=', 'quiet', 'source', 'style=',
'username=', 'verbose', 'version'))
except getopt.error, exc:
print 'youget: %s, try youget -h for a list of all the options' % str(exc)
sys.exit(1)
for opt, arg in opts:
if opt in ('-d', '--credpath'):
self.credpath = arg
elif opt in ('--delete', ):
self.cleanup = True
elif opt in ('--download-all', ):
self.downloadall = True
elif opt in ('--filter', ):
self.filter = arg
self.downloadall = True
elif opt in ('-h', '--help'):
self.usage()
print
self.help()
sys.exit(0)
elif opt in ('-l', '--list'):
self.list = True
self.downloadall = True
elif opt in ('-n', '--dry-run'):
self.dryrun = True
elif opt in ['-p', '--password']:
self.password = arg
elif opt in ('-q', '--quiet'):
self.quiet = True
elif opt in ('--source'):
self.source = True
elif opt in ('-s', '--style'):
self.style = arg
elif opt in ['-u', '--username']:
self.username = arg
elif opt in ('-v', '--verbose'):
self.verbose = self.verbose + 1
elif opt in ('--version', ):
self.version()
sys.exit(0)
if len(args) < 1:
self.usage()
print
self.help()
sys.exit(1)
self.uri = args[0]
if len(args) == 2:
self.destination = args[1]
else:
self.destination = os.getcwd()
if self.quiet:
self.verbose = 0
if self.verbose >= 3:
print 'Verbosity set to level %d' % (self.verbose - 1)
def version(self):
print 'youget %s' % VERSION
print 'Written by Dag Wieers <dag@wieers.com>'
print
print 'platform %s/%s' % (os.name, sys.platform)
print 'python %s' % sys.version
print
print 'build revision $Rev: 4786 $'
def usage(self):
print 'usage: youget [options] URL'
def help(self):
print '''Download packages from Yast Online Update (YOU)
youget options:
-d, --credpath=dir credentials directory
--delete delete files that are not on the sender side
--download-all download all package versions available
--filter filter packages based on regexp
-l, --list list the available packages
-n, --dry-run show what would have been done
-q, --quiet minimal output
--source download source packages
-v, --verbose increase verbosity
-vv, -vvv, -vvvv.. increase verbosity more
'''
class Set:
def __init__(self):
self.list = []
def add(self, input):
if input not in self.list:
self.list.append(input)
def delete(self, input):
if input in self.list:
self.list.removed(input)
def difference(self, other):
newlist = Set()
for element in self.list:
if element not in other.list:
newlist.add(element)
return newlist
def sort(self):
return self.list.sort()
def __str__(self):
return '\n\t' + '\n\t'.join([element[0] for element in self.list])
def __len__(self):
return len(self.list)
class MirrorException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def error(level, str):
"Output error message"
if level <= op.verbose:
sys.stderr.write('youget: %s\n' % str)
def info(level, str):
"Output info message"
if level <= op.verbose:
sys.stdout.write('%s\n' % str)
def die(ret, str):
"Print error and exit with errorcode"
error(0, str)
sys.exit(ret)
def filelist(top):
flist = []
for root, dirs, files in os.walk(top):
for file in files:
flist.append(os.path.join(root, file))
flist.sort()
return flist
def remove(file):
"Remove files or directories"
if isinstance(file, types.StringType):
if op.dryrun:
return
if os.path.islink(file):
os.unlink(file)
elif os.path.isdir(file):
try:
os.rmdir(file)
except:
os.path.walk(file, removedir, ())
os.rmdir(file)
elif os.path.isfile(file) or os.path.islink(file):
os.unlink(file)
else:
for f in file:
remove(f)
def mkdir(path):
"Create a directory, and parents if needed"
if op.dryrun:
return
if os.path.islink(path):
os.unlink(path)
if not os.path.exists(path):
os.makedirs(path)
def httpget(opener, file, url, path=None):
"Download file from url to local path"
if path:
mkdir(path)
else:
path = os.getcwd()
if os.path.dirname(file):
mkdir(os.path.join(path, os.path.dirname(file)))
fdin = opener.open(os.path.join(url, file))
fdout = open(os.path.join(path, file), 'w')
fdout.write(fdin.read())
fdin.close()
fdout.close()
def mirroryou(url, path):
'Check username/password and YOU mirror style'
### See if we find mcookie/partnernet in credpath
if not op.username and not op.password:
try:
op.username = open(os.path.join(op.credpath, 'deviceid')).read().rstrip().rstrip('\0')
op.password = open(os.path.join(op.credpath, 'secret')).read().rstrip().rstrip('\0')
op.style = 'sles10'
except:
info(3, 'Credentials directory %s does not contain deviceid and secret files. (SLES10)' % op.credpath)
### See if we find mcookie/partnernet in credpath
if not op.username and not op.password:
try:
op.username = open(os.path.join(op.credpath, 'mcookie')).read().rstrip().rstrip('\0')
op.password = open(os.path.join(op.credpath, 'partnernet')).read().rstrip().rstrip('\0')
op.style = 'nld9'
except:
info(3, 'Credentials directory %s does not contain mcookie and partnernet files. (NLD9)' % op.credpath)
if op.credpath and not op.username and not op.password:
die(2, 'No credentials found in %s.' % op.credpath)
if not op.username:
op.username = raw_input('YOU Username: ')
if op.username and not op.password:
op.password = getpass.getpass('YOU Password for user %s: ' % op.username)
if op.style == 'sles10':
mirroryou_sles10(url, path)
elif op.style == 'nld9':
mirroryou_nld9(url, path)
else:
mirroryou_sles10(url, path)
def mirroryou_sles10(url, path):
'Mirror a channel from YOU (SLES10 style)'
info(3, 'Using username %s with password %s.' % (op.username, op.password))
### Setting up connection
host = urlparse.urlparse(url)[1]
auth_handler = urllib2.HTTPDigestAuthHandler()
auth_handler.add_password('Express', host, op.username, op.password)
opener = urllib2.build_opener(auth_handler)
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
### Download repodata for this channel
info(2, 'Downloading packagelist and metadata from %s' % url)
### FIXME: check repomd.xml to see if any of the files have been updated
# httpget(opener, 'repodata/repomd.xml.asc', url, path)
# httpget(opener, 'repodata/repomd.xml.key', url, path)
# httpget(opener, 'repodata/repomd.xml', url, path)
# httpget(opener, 'repodata/filelists.xml.gz', url, path)
httpget(opener, 'repodata/primary.xml.gz', url, path)
# httpget(opener, 'repodata/patches.xml', url, path)
### Parse packagelist
fd = gzip.open(os.path.join(path, 'repodata/primary.xml.gz'), 'r')
tree = ElementTree.ElementTree(file=fd)
root = tree.getroot()
package_list = Set()
for elem in root.getiterator('{http://linux.duke.edu/metadata/common}package'):
pkgname = elem.find('{http://linux.duke.edu/metadata/common}location').get('href')
pkgsize = int(elem.find('{http://linux.duke.edu/metadata/common}size').get('package'))
package_list.add((pkgname, pkgsize))
fd.close()
package_list.sort()
### Download packages from the packagelist
for filename, filesize in package_list.list:
### Filter packagelist
if op.filter and not fnmatch.fnmatch(filename, op.filter):
info(4, 'Packages %s excluded by filter' % filename)
continue
### List only files if requested
if op.list:
info(0, filename)
continue
### If file (or symlink target) exists
if os.path.isfile(os.path.join(path, filename)):
stat = os.stat(os.path.join(path, filename))
if stat.st_size == filesize:
info(3, 'File %s is already in %s' % (filename, path))
continue
else:
info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize))
remove(os.path.join(path, filename))
### If symlink target does not exist, remove symlink
elif os.path.islink(os.path.join(path, filename)):
remove(os.path.join(path, filename))
if op.dryrun:
info(1, 'Not downloading package %s' % filename)
continue
info(2, 'Download %s (%s)' % (filename, filesize))
httpget(opener, filename, url, path)
### Remove packages on the receiver side that are not on the sender side
if op.cleanup:
### Collect receiver side
receiver = Set()
for file in filelist(os.path.join(path, 'rpm')):
filename = file.split(path + '/')[1]
filesize = os.stat(file).st_size
receiver.add((filename, filesize))
receiver.sort()
### Collect sender side
sender = package_list
### Remove difference between receiver and sender
cleanse = receiver.difference(sender)
for filename, filesize in cleanse.list:
info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize))
remove(os.path.join(path, filename))
def mirroryou_nld9(url, path):
'Mirror a channel from YOU (NLD9 style)'
info(3, 'Using username %s with password %s.' % (op.username, op.password))
### Setting up connection
host = urlparse.urlparse(url)[1]
auth_handler = urllib2.HTTPDigestAuthHandler()
auth_handler.add_password('Express', host, op.username, op.password)
opener = urllib2.build_opener(auth_handler)
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
### Download repodata for this channel
info(2, 'Downloading packagelist from %s' % url)
httpget(opener, 'packageinfo.xml.gz', url, path)
### Parse packagelist
fd = gzip.open(os.path.join(path, 'packageinfo.xml.gz'), 'r')
tree = ElementTree.ElementTree(file=fd)
root = tree.getroot()
package_list = Set()
for elem in root.getiterator('package'):
pkgname = elem.findtext('history/update/filename')
pkgsize = int(elem.findtext('history/update/filesize'))
package_list.add((pkgname, pkgsize))
fd.close()
package_list.sort()
### Download packages from the packagelist
for filename, filesize in package_list.list:
### Filter packagelist
if op.filter and not fnmatch.fnmatch(filename, op.filter):
info(4, 'Packages %s excluded by filter' % filename)
continue
### List only files if requested
if op.list:
info(0, filename)
continue
### If file (or symlink target) exists
if os.path.isfile(os.path.join(path, filename)):
stat = os.stat(os.path.join(path, filename))
if stat.st_size == filesize:
info(3, 'File %s is already in %s' % (filename, path))
continue
else:
info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize))
remove(os.path.join(path, filename))
### If symlink target does not exist, remove symlink
elif os.path.islink(os.path.join(path, filename)):
remove(os.path.join(path, filename))
if op.dryrun:
info(1, 'Not downloading package %s' % filename)
continue
info(2, 'Download %s (%s)' % (filename, filesize))
httpget(opener, filename, url, path)
### Remove packages on the receiver side that are not on the sender side
if op.cleanup:
### Collect receiver side
receiver = Set()
for file in filelist(os.path.join(path, 'rpm')):
filename = file.split(path + '/')[1]
filesize = os.stat(file).st_size
receiver.add((filename, filesize))
receiver.sort()
### Collect sender side
sender = package_list
### Remove difference between receiver and sender
cleanse = receiver.difference(sender)
for filename, filesize in cleanse.list:
info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize))
remove(os.path.join(path, filename))
def main():
try:
mirroryou(op.uri, op.destination)
except Exception, e:
die(1, e)
### Unbuffered sys.stdout
sys.stdout = os.fdopen(1, 'w', 0)
sys.stderr = os.fdopen(2, 'w', 0)
### Workaround for python <= 2.2.1
try:
True, False
except NameError:
True = 1
False = 0
### Main entrance
if __name__ == '__main__':
exitcode = 0
op = Options(sys.argv[1:])
try:
main()
except KeyboardInterrupt, e:
die(6, 'Exiting on user request')
sys.exit(exitcode)
# vim:ts=4:sw=4:et
Jump to Line
Something went wrong with that request. Please try again.