diff --git a/config/dists/sles10.conf b/config/dists/sles10.conf index 10c8e0d..27cb0f0 100644 --- a/config/dists/sles10.conf +++ b/config/dists/sles10.conf @@ -7,10 +7,11 @@ [sles10] name = SuSE Linux Enterprise Server $release SP1 ($arch) release = 10 -arch = i386 ia64 ppc s390x x86_64 +arch = i586 ia64 ppc s390x x86_64 ### ISO images iso = SLES-$release-SP1-$arch-CD?.iso SLES-$release-SP1-$arch-DVD?.iso ### Additional repositories -you = file:///var/yast2/you/ +updates = you://update.novell.com/repo/$RCE/SLES10-Updates/sles-10-$arch/ +sdk = you://update.novell.com/repo/$RCE/SLE10-SDK-Updates/sles-10-$arch/ diff --git a/docs/you-support.txt b/docs/you-support.txt index 1a49e08..e36ef13 100644 --- a/docs/you-support.txt +++ b/docs/you-support.txt @@ -11,10 +11,10 @@ various Novell/SUSE distributions and architectures. The youget python script needs cElementTree support. -== Novell update credentials -To be able to download updates from update.novell.com (or another YOU server) -you need to copy the files _/etc/ximian/mcookie_ and _/etc/ximian/partnernet_ -from a registered/activated system to your +$srcdir/$dist-$arch/+ directory. +== Novell update credentials for NLD9 (and alike) +To be able to download updates from update.novell.com you need to copy the +files _/etc/ximian/mcookie_ and _/etc/ximian/partnernet_ from a registered +system to your +$srcdir/$dist-$arch/+ directory. So for NLD9, you would have eg: @@ -24,6 +24,19 @@ So for NLD9, you would have eg: These are used by youget to authenticate with the YOU server. +== Novell update credentials for SLES10 (and alike) +To be able to download updates from nu.novell.com you need to copy the +files _/etc/zmd/deviceid_ and _/etc/zmd/secret_ from a registered +system to your +$srcdir/$dist-$arch/+ directory. + +So for SLES10, you would have eg: + + /var/mrepo/sles10-x86_64/deviceid + /var/mrepo/sles10-x86_64/secret + +These are used by youget to authenticate with the YOU server. + + == YOU configuration Having done that, you can enable multiple channels for these credentials by adding a configuration like the one below: @@ -32,11 +45,23 @@ adding a configuration like the one below: [nld9] name = Novell Linux Desktop $release SP3 ($arch) release = 9 -arch = i386 +arch = i586 iso = NLD$release-SP3-$arch-CD?.iso -updates = you://update.novell.com/data/$RCE/nld9/getPackage/nld-9-i586/ -extras = you://update.novell.com/data/$RCE/nld9-extras/getPackage/nld-9-i586/ -sdk = you://update.novell.com/data/$RCE/nld9-sdk/getPackage/nld-9-i586/ +updates = you://update.novell.com/data/$RCE/nld9/getPackage/nld-9-$arch/ +extras = you://update.novell.com/data/$RCE/nld9-extras/getPackage/nld-9-$arch/ +sdk = you://update.novell.com/data/$RCE/nld9-sdk/getPackage/nld-9-$arch/ +---- + +or + +---- +[sles10] +name = SuSE Linux Enterprise Server $release SP1 ($arch) +release = 10 +arch = i586 ia64 ppc s390x x86_64 +iso = SLES-$release-SP1-$arch-CD?.iso SLES-$release-SP1-$arch-DVD?.iso +updates = you://update.novell.com/repo/$RCE/SLES10-Updates/sles-10-$arch/ +sdk = you://update.novell.com/repo/$RCE/SLE10-SDK-Updates/sles-10-$arch/ ---- After that, mrepo should be able to successfully log on and download diff --git a/mrepo.spec b/mrepo.spec index bcc1761..50fc593 100644 --- a/mrepo.spec +++ b/mrepo.spec @@ -101,8 +101,9 @@ fi %config(noreplace) %{_sysconfdir}/mrepo.conf.d/ %config %{_initrddir}/mrepo %{_bindir}/gensystemid -%{_bindir}/rhnget %{_bindir}/mrepo +%{_bindir}/rhnget +%{_bindir}/youget %{_datadir}/mrepo/ %{_localstatedir}/cache/mrepo/ %{_localstatedir}/www/mrepo/ diff --git a/rhnget b/rhnget index 0457f09..f35903f 100755 --- a/rhnget +++ b/rhnget @@ -413,7 +413,7 @@ def mirrorrhn(url, path): for pkg in package_list: ### FIXME: Check if not already on ISO-file or repository as well filename = '%s-%s-%s.%s.rpm' % (pkg[0], pkg[1], pkg[2], pkg[4]) - filesize = pkg[5] + filesize = int(pkg[5]) ### Filter packagelist if op.filter and not fnmatch.fnmatch(filename, op.filter): @@ -428,7 +428,7 @@ def mirrorrhn(url, path): ### If file (or symlink target) exists if os.path.isfile(os.path.join(path, filename)): stat = os.stat(os.path.join(path, filename)) - if stat.st_size == int(filesize): + if stat.st_size == filesize: info(3, 'File %s is already in %s' % (filename, path)) continue else: diff --git a/youget b/youget index 95af059..331ee62 100755 --- a/youget +++ b/youget @@ -14,6 +14,9 @@ ### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. ### Copyright 2004-2006 Dag Wieers +### For SLES10 implementation, see: +### http://lists.suse.com/archive/suse-sles-e/2006-Aug/0161.html + import os, sys, shutil, getopt, ConfigParser, urlparse, types import signal, xmlrpclib, getpass, glob, fnmatch, urllib2 import gzip @@ -35,13 +38,14 @@ class Options: self.quiet = False self.password = None self.username = None + self.style = 'sles10' self.source = False self.verbose = 1 try: opts, args = getopt.getopt (args, 'd:hlnqp:s:u:v', ('credpath=', 'delete', 'download-all', 'dryrun', 'filter=', 'help', 'list', - 'password=', 'quiet', 'source', 'systemid=', 'username=', 'verbose', 'version' )) + 'password=', 'quiet', 'source', 'style=', 'username=', 'verbose', 'version' )) except getopt.error, exc: print 'youget: %s, try youget -h for a list of all the options' % str(exc) sys.exit(1) @@ -72,8 +76,8 @@ class Options: self.quiet = True elif opt in ('--source'): self.source = True - elif opt in ('-s', '--systemid'): - self.systemid = os.path.abspath(arg) + elif opt in ('-s', '--style'): + self.style = arg elif opt in ['-u', '--username']: self.username = arg elif opt in ('-v', '--verbose'): @@ -90,32 +94,6 @@ class Options: self.uri = args[0] - if not self.username and not self.password: - if self.credpath: - try: - self.username = open(os.path.join(self.credpath, 'mcookie')).read().rstrip().rstrip('\0') - self.password = open(os.path.join(self.credpath, 'partnernet')).read().rstrip().rstrip('\0') - except: - die(1, 'Credentials directory %s does not contain mcookie and partnernet files.' % op.credpath) - elif os.path.isdir('/etc/ximian'): - try: - self.username = open('/etc/ximian/mcookie').read().rstrip().rstrip('\0') - self.password = open('/etc/ximian/partnernet').read().rstrip().rstrip('\0') - except: - die(1, 'Credentials directory /etc/ximian does not contain mcookie and partnernet files.') - elif os.path.isdir('/etc/zmd'): - try: - self.username = open('/etc/zmd/mcookie').read().rstrip().rstrip('\0') - self.password = open('/etc/zmd/partnernet').read().rstrip().rstrip('\0') - except: - die(1, 'Credentials directory /etc/zmd does not contain mcookie and partnernet files.') - - if not self.username: - self.username = raw_input('YOU Username: ') - - if self.username and not self.password: - self.password = getpass.getpass('YOU Password for user %s: ' % self.username) - if len(args) == 2: self.destination = args[1] else: @@ -204,6 +182,14 @@ def die(ret, str): error(0, str) sys.exit(ret) +def filelist(top): + flist = [] + for root, dirs, files in os.walk(top): + for file in files: + flist.append(os.path.join(root, file)) + flist.sort() + return flist + def remove(file): "Remove files or directories" if isinstance(file, types.StringType): @@ -223,23 +209,169 @@ def remove(file): for f in file: remove(f) +def mkdir(path): + "Create a directory, and parents if needed" + if op.dryrun: + return + if os.path.islink(path): + os.unlink(path) + if not os.path.exists(path): + os.makedirs(path) + +def httpget(opener, file, url, path=None): + "Download file from url to local path" + if path: + mkdir(path) + else: + path = os.getcwd() + if os.path.dirname(file): + mkdir(os.path.join(path, os.path.dirname(file))) + fdin = opener.open(os.path.join(url, file)) + fdout = open(os.path.join(path, file), 'w') + fdout.write(fdin.read()) + fdin.close() + fdout.close() + def mirroryou(url, path): - 'Mirror a channel from YOU' + 'Check username/password and YOU mirror style' + + ### See if we find mcookie/partnernet in credpath + if not op.username and not op.password: + try: + op.username = open(os.path.join(op.credpath, 'deviceid')).read().rstrip().rstrip('\0') + op.password = open(os.path.join(op.credpath, 'secret')).read().rstrip().rstrip('\0') + op.style = 'sles10' + except: + info(3, 'Credentials directory %s does not contain deviceid and secret files. (SLES10)' % op.credpath) + + ### See if we find mcookie/partnernet in credpath + if not op.username and not op.password: + try: + op.username = open(os.path.join(op.credpath, 'mcookie')).read().rstrip().rstrip('\0') + op.password = open(os.path.join(op.credpath, 'partnernet')).read().rstrip().rstrip('\0') + op.style = 'nld9' + except: + info(3, 'Credentials directory %s does not contain mcookie and partnernet files. (NLD9)' % op.credpath) + + if op.credpath and not op.username and not op.password: + die(2, 'No credentials found in %s.' % op.credpath) + + if not op.username: + op.username = raw_input('YOU Username: ') + + if op.username and not op.password: + op.password = getpass.getpass('YOU Password for user %s: ' % op.username) + + if op.style == 'sles10': + mirroryou_sles10(url, path) + elif op.style == 'nld9': + mirroryou_nld9(url, path) + else: + mirroryou_sles10(url, path) + +def mirroryou_sles10(url, path): + 'Mirror a channel from YOU (SLES10 style)' info(3, 'Using username %s with password %s.' % (op.username, op.password)) - ### Download packagelist for this channel - info(2, 'Downloading packagelist from %s' % url) + ### Setting up connection host = urlparse.urlparse(url)[1] auth_handler = urllib2.HTTPDigestAuthHandler() auth_handler.add_password('Express', host, op.username, op.password) opener = urllib2.build_opener(auth_handler) opener.addheaders = [('User-agent', 'Mozilla/5.0')] - fdin = opener.open(os.path.join(url, 'packageinfo.xml.gz')) - fdout = open(os.path.join(path,'packageinfo.xml.gz'), 'w') - fdout.write(fdin.read()) - fdin.close() - fdout.close() + + ### Download repodata for this channel + info(2, 'Downloading packagelist and metadata from %s' % url) + ### FIXME: check repomd.xml to see if any of the files have been updated +# httpget(opener, 'repodata/repomd.xml.asc', url, path) +# httpget(opener, 'repodata/repomd.xml.key', url, path) +# httpget(opener, 'repodata/repomd.xml', url, path) +# httpget(opener, 'repodata/filelists.xml.gz', url, path) + httpget(opener, 'repodata/primary.xml.gz', url, path) +# httpget(opener, 'repodata/patches.xml', url, path) + + ### Parse packagelist + fd = gzip.open(os.path.join(path, 'repodata/primary.xml.gz'), 'r') + tree = ElementTree.ElementTree(file=fd) + root = tree.getroot() + package_list = Set() + for elem in root.getiterator('{http://linux.duke.edu/metadata/common}package'): + pkgname = elem.find('{http://linux.duke.edu/metadata/common}location').get('href') + pkgsize = int(elem.find('{http://linux.duke.edu/metadata/common}size').get('package')) + package_list.add( (pkgname, pkgsize) ) + fd.close() + package_list.sort() + + ### Download packages from the packagelist + for filename, filesize in package_list.list: + + ### Filter packagelist + if op.filter and not fnmatch.fnmatch(filename, op.filter): + info(4, 'Packages %s excluded by filter' % filename) + continue + + ### List only files if requested + if op.list: + info(0, filename) + continue + + ### If file (or symlink target) exists + if os.path.isfile(os.path.join(path, filename)): + stat = os.stat(os.path.join(path, filename)) + if stat.st_size == filesize: + info(3, 'File %s is already in %s' % (filename, path)) + continue + else: + info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize)) + remove(os.path.join(path, filename)) + + ### If symlink target does not exist, remove symlink + elif os.path.islink(os.path.join(path, filename)): + remove(os.path.join(path, filename)) + + if op.dryrun: + info(1, 'Not downloading package %s' % filename) + continue + + info(2, 'Download %s (%s)' % (filename, filesize)) + httpget(opener, filename, url, path) + + ### Remove packages on the receiver side that are not on the sender side + if op.cleanup: + + ### Collect receiver side + receiver = Set() + for file in filelist(os.path.join(path, 'rpm')): + filename = file.split(path+'/')[1] + filesize = os.stat(file).st_size + receiver.add( (filename, filesize) ) + receiver.sort() + + ### Collect sender side + sender = package_list + + ### Remove difference between receiver and sender + cleanse = receiver.difference(sender) + for filename, filesize in cleanse.list: + info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize)) + remove(os.path.join(path, filename)) + +def mirroryou_nld9(url, path): + 'Mirror a channel from YOU (NLD9 style)' + + info(3, 'Using username %s with password %s.' % (op.username, op.password)) + + ### Setting up connection + host = urlparse.urlparse(url)[1] + auth_handler = urllib2.HTTPDigestAuthHandler() + auth_handler.add_password('Express', host, op.username, op.password) + opener = urllib2.build_opener(auth_handler) + opener.addheaders = [('User-agent', 'Mozilla/5.0')] + + ### Download repodata for this channel + info(2, 'Downloading packagelist from %s' % url) + httpget(opener, 'packageinfo.xml.gz', url, path) ### Parse packagelist fd = gzip.open(os.path.join(path,'packageinfo.xml.gz'), 'r') @@ -285,22 +417,17 @@ def mirroryou(url, path): continue info(2, 'Download %s (%s)' % (filename, filesize)) - fdin = opener.open(os.path.join(url, filename)) - fdout = open(os.path.join(path, filename), 'w') - fdout.write(fdin.read()) - fdin.close() - fdout.close() + httpget(opener, filename, url, path) ### Remove packages on the receiver side that are not on the sender side if op.cleanup: ### Collect receiver side receiver = Set() - for file in glob.glob(os.path.join(path, '*.rpm')): - if os.path.exists(file): - filename = os.path.basename(file) - filesize = os.stat(file).st_size - receiver.add( (filename, filesize) ) + for file in filelist(os.path.join(path, 'rpm')): + filename = file.split(path+'/')[1] + filesize = os.stat(file).st_size + receiver.add( (filename, filesize) ) receiver.sort() ### Collect sender side