rhnget

#!/usr/bin/python

### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU Library General Public License as published by
### the Free Software Foundation; version 2 only
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU Library General Public License for more details.
###
### You should have received a copy of the GNU Library General Public License
### along with this program; if not, write to the Free Software
### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
### Copyright 2004-2006 Dag Wieers <dag@wieers.com>

import os, sys, shutil, getopt, ConfigParser, urlparse, types
import signal, xmlrpclib, getpass, glob, fnmatch

if os.path.exists('/usr/share/mrepo/up2date_client/'):
    sys.path.insert(-1, '/usr/share/mrepo/')
    sys.path.insert(-1, '/usr/share/mrepo/up2date_client/')
elif os.path.exists('/usr/share/rhn/up2date_client/'):
    sys.path.insert(-1, '/usr/share/rhn/')
    sys.path.insert(-1, '/usr/share/rhn/up2date_client/')
else:
    print >>sys.stderr, 'rhnget: up2date libraries are not installed. Aborting execution'
    sys.exit(1)

from up2date_client import config, rpcServer, wrapperUtils, up2dateErrors, repoDirector
from rhn import rpclib

cfg = {}
loginInfo = {}
rd = None

__version__ = "$Revision: 4786 $"
# $Source$

VERSION = '0.8.4svn'

### Register rhn and rhns as a known schemes
for scheme in ('rhn', 'rhns'):
    urlparse.uses_netloc.insert(0, scheme)
    urlparse.uses_query.insert(0, scheme)

class Options:
    def __init__(self, args):
        self.cleanup = False
        self.downloadall = False
        self.dryrun = False
        self.filter = None
        self.list = None
        self.quiet = False
        self.rhnpassword = None
        self.rhnrelease = None
        self.rhnusername = None
        self.source = False
        self.systemid = '/etc/sysconfig/rhn/systemid'
        self.verbose = 1

        try:
            opts, args = getopt.getopt (args, 'hlnqp:r:s:u:v',
                ('delete', 'download-all', 'dryrun', 'filter=', 'help', 'list', 'password=', 'quiet',
                 'release=', 'source', 'systemid=', 'username=', 'verbose', 'version' ))
        except getopt.error, exc:
            print 'rhnget: %s, try rhnget -h for a list of all the options' % str(exc)
            sys.exit(1)

        for opt, arg in opts:
            if opt in ('--delete', ):
                self.cleanup = True
            elif opt in ('--download-all', ):
                self.downloadall = True
            elif opt in ('--filter', ):
                self.filter = arg
                self.downloadall = True
            elif opt in ('-h', '--help'):
                self.usage()
                print
                self.help()
                sys.exit(0)
            elif opt in ('-l', '--list'):
                self.list = True
                self.downloadall = True
            elif opt in ('-n', '--dry-run'):
                self.dryrun = True
            elif opt in ['-p', '--password']:
                self.rhnpassword = arg
            elif opt in ('-q', '--quiet'):
                self.quiet = True
            elif opt in ('-r', '--release'):
                self.rhnrelease = arg
            elif opt in ('--source', ):
                self.source = True
            elif opt in ('-s', '--systemid'):
                self.systemid = os.path.abspath(arg)
            elif opt in ['-u', '--username']:
                self.rhnusername = arg
            elif opt in ('-v', '--verbose'):
                self.verbose = self.verbose + 1
            elif opt in ('--version', ):
                self.version()
                sys.exit(0)

        if len(args) < 1:
            self.usage()
            print
            self.help()
            sys.exit(1)

        self.uri = args[0]

        if len(args) == 2:
            self.destination = args[1]
        else:
            self.destination = os.getcwd()

        if self.quiet:
            self.verbose = 0

        if self.verbose >= 3:
            print 'Verbosity set to level %d' % (self.verbose - 1)

    def version(self):
        print 'rhnget %s' % VERSION
        print 'Written by Dag Wieers <dag@wieers.com>'
        print
        print 'platform %s/%s' % (os.name, sys.platform)
        print 'python %s' % sys.version
        print
        print 'build revision $Rev: 4786 $'

    def usage(self):
        print 'usage: rhnget [options] rhns://server/channel destination-path'

    def help(self):
        print '''Download packages from Red Hat Network (RHN)

rhnget options:
      --delete            delete files that are not on the sender side
      --download-all      download all package versions available
      --filter            filter packages based on regexp
  -l, --list              list the available packages
  -n, --dry-run           show what would have been done
  -q, --quiet             minimal output
  -r, --release=release   specify the RHN release (if different from the systemid)
      --source            download source packages
  -s, --systemid=file     systemid to use
  -v, --verbose           increase verbosity
  -vv, -vvv, -vvvv..      increase verbosity more
'''

class Set:
    def __init__(self):
        self.list = []

    def add(self, input):
        if input not in self.list:
            self.list.append(input)
    
    def delete(self, input):
        if input in self.list:
            self.list.removed(input)
    
    def difference(self, other):
        newlist = Set()
        for element in self.list:
            if element not in other.list:
                newlist.add(element)
        return newlist

    def sort(self):
        return self.list.sort()
    
    def __str__(self):
        return '\n\t' + '\n\t'.join([element[0] for element in self.list])

    def __len__(self):
        return len(self.list)

class MirrorException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

def error(level, str):
    "Output error message"
    if level <= op.verbose:
        sys.stderr.write('rhnget: %s\n' % str)

def info(level, str):
    "Output info message"
    if level <= op.verbose:
        sys.stdout.write('%s\n' % str)

def die(ret, str):
    "Print error and exit with errorcode"
    error(0, str)
    sys.exit(ret)

def remove(file):
    "Remove files or directories"
    if isinstance(file, types.StringType):
        if op.dryrun:
            return
        if os.path.islink(file):
            os.unlink(file)
        elif os.path.isdir(file):
            try:
                os.rmdir(file)
            except:
                os.path.walk(file, removedir, ())
                os.rmdir(file)
        elif os.path.isfile(file) or os.path.islink(file):
            os.unlink(file)
    else:
        for f in file:
            remove(f)

def removedir(void, dir, files):
    for file in files:
        remove(os.path.join(dir, file))

def mkdir(path):
    "Create a directory, and parents if needed"
    if op.dryrun:
        return
    if os.path.islink(path):
        os.unlink(path)
    if not os.path.exists(path):
        os.makedirs(path)

def rhnlogin(url, path, force=False):
    'Log on to RHN and return cfg, loginInfo and systemid'
    global cfg, loginInfo, rd, config, rpcServer

    ### Look for the usual suspects
    if os.path.isfile(op.systemid):
        systemidpath = op.systemid
    elif os.path.isfile('/etc/sysconfig/rhn/systemid'):
        systemidpath = '/etc/sysconfig/rhn/systemid'
    else:
        info(1, 'No RHN systemid found, skipping download.')
        return
    info(3, 'Using RHN systemid from %s' % systemidpath)

    systemid = open(systemidpath).read()

    cfg['systemIdPath'] = systemidpath
    cfg = config.initUp2dateConfig()
    cfg['systemIdPath'] = systemidpath
    cfg['storageDir'] = path
    cfg['retrieveOnly'] = 1
    cfg['keepAfterInstall'] = 1
    cfg['noReboot'] = 1
    cfg['useRhn'] = 1
    cfg['showChannels'] = 1
    cfg['showAvailablePackages'] = 1
    cfg['isatty'] = 1
    cfg['networkRetries'] = 3
#   cfg['headerFetchCount'] = 20
    cfg['enableProxy'] = 0
    cfg['enableProxyAuth'] = 0
    cfg['httpProxy'] = ''
    cfg['proxyUser='] = ''
    cfg['proxyPassword'] = ''
    cfg["sslCACert"] = '/usr/share/rhn/RHNS-CA-CERT'

    ### Override the version if forced in mrepo configuration (to allow single systemid usage)
    if op.rhnrelease:
        cfg['versionOverride'] = op.rhnrelease
    else:
        cfg['versionOverride'] = rpclib.xmlrpclib.loads(systemid)[0][0]['os_release']
    info(3, 'Using RHN release %s' % cfg['versionOverride'])

#   if op.arch:
#       cfg['forceArch'] = '%s-redhat-linux' % op.arch

    ### Modify the logfile in case we have no rights to write in /var/log/up2date (non-root)
    if os.access('/var/log/up2date', os.W_OK):
        cfg['logFile'] = '/var/log/up2date'
    else:
        cfg['logFile'] = os.path.expanduser('~/up2date.log')

    ### If we're not targetting the default RHN server, change the location
    rhnscheme, rhnserver, t, t, t, t = urlparse.urlparse(url)
    if not rhnserver:
        rhnserver = 'xmlrpc.rhn.redhat.com'

    cfg['noSSLServerURL'] = 'http://%s/XMLRPC' % rhnserver
    if rhnscheme == 'rhn':
        cfg['serverURL'] = 'http://%s/XMLRPC' % rhnserver
    else:
        cfg['serverURL'] = 'https://%s/XMLRPC' % rhnserver

    ### Get proxy information from environment and set up2date config accordingly
    proxy = None
    if os.environ.has_key('http_proxy') and rhnscheme == 'rhn':
        t, proxy, t, t, t, t = urlparse.urlparse(os.environ['http_proxy'])
    elif os.environ.has_key('https_proxy') and rhnscheme == 'rhns':
        t, proxy, t, t, t, t = urlparse.urlparse(os.environ['https_proxy'])
    if proxy:
        cfg['enableProxy'] = 1
        cfg['httpProxy'] = proxy
        info(4, 'Setting proxy for %s to %s' % (rhnscheme, proxy))
        ### FIXME: Implement proxy authentication
#       if proxy.username and proxy.password:
#           cfg['enableProxyAuth'] = 1
#           cfg['proxyPassword'] = proxy.password
#           cfg['proxyUser='] = proxy.username

    ### Set debugging information to something very high (there seems to be no granularity)
    if op.verbose >= 3:
        cfg['debug'] = 1

    info(4, '\nBEFORE LOGIN: logininfo: %s\n' % loginInfo)
    try:
        server = rpcServer.getServer()
        li = rpcServer.doCall(server.up2date.login, systemid)
        loginInfo.update(li)
    except rpclib.Fault, f:
        error(1, 'Error logging in with systemid %s. %s' % (systemidpath, f.faultString))
        return None
    info(4, '\nAFTER LOGIN: logininfo: %s\n' % loginInfo)

    return systemid

def rhngetchannel(channels, label):
    'Return the channel with given label, if found'
    for c in channels:
        if isinstance(c, types.ListType):
            l, v = c[0], c[1]
        else:
            l, v = c['label'], c['version']
        if l == label:
            return {
                'label': l,
                'version': v,
                'type': 'up2date',
                'url': cfg['serverURL'],
            }
    return None

def mirrorrhn(url, path):
    'Mirror a channel from RHN'
    global cfg, loginInfo, rd, repoDirector, rpcServer

    t, t, label, t, t, t = urlparse.urlparse(url)
    label = label.strip('/')

    ### Log on to RHN
    systemid = rhnlogin(url, path)
    if not systemid:
        return

    mkdir(cfg['storageDir'])

    ### Try to find a channel with label
    channel = rhngetchannel(loginInfo.get('X-RHN-Auth-Channels'), label)
    if not channel:
#       raise(Exception('Error system not subscribe to channel %s, skipping.' % label))
        if not op.rhnusername:
            op.rhnusername = raw_input('RHN Username: ')

        if op.rhnusername and not op.rhnpassword:
            op.rhnpassword = getpass.getpass('RHN Password for user %s: ' % op.rhnusername)

        if op.rhnusername and op.rhnpassword:
            try:
                server = rpcServer.getServer()
                channels = rpcServer.doCall(server.up2date.subscribeChannels, systemid, (label,), op.rhnusername, op.rhnpassword)
            except rpclib.Fault, f:
                raise(Exception('Error subscribing to channel %s, skipping.%s' % (label, f.faultString)))
            systemid = rhnlogin(url, path, force=True)
            if not systemid:
                return
            info(4, '\nAFTER SUBSC: logininfo: %s\n' % loginInfo)
            channel = rhngetchannel(loginInfo.get('X-RHN-Auth-Channels'), label)
            if not channel:
                raise(Exception('Failed to subscribe RHN id to channel %s, skipping.' % label))
        else:
            raise(Exception('No RHN username or password supplied. Please add channel %s on RHN website manually. Skipping.' % label))

    ### Download packagelist for this channel
    try:
        repos = repoDirector.initRepoDirector()
    except xmlrpclib.Fault, f:
        raise(MirrorException('Problem setting up XML communication for channel %s.\n%s' % (label, f.faultString)))
        return
    except up2dateErrors.ServerCapabilityError, e:
        raise(MirrorException('Problem negotiating capabilities for channel %s.\n%s' % (label, e)))
        return

    try:
        if op.downloadall == True:
            package_list, type = rpcServer.doCall(repos.listAllPackages, channel, None, None)
        else:
            package_list, type = rpcServer.doCall(repos.listPackages, channel, None, None)
    except rpclib.Fault, f:
        raise(MirrorException('Error listing packages from channel %s. Skipping. %s' % (label, f.faultString)))
    except up2dateErrors.CommunicationError, e:
        raise(MirrorException('Error listing packages from channel %s. Skipping.\n%s' % (label, e)))
    except KeyError, e:
        if e == "'up2date'":
            raise(MirrorException('Missing up2date entry in /etc/sysconfig/rhn/sources.'))
        else:
            raise(MirrorException('Unknown error that needs more debugging occured with channel %s. Skipping.\n%s' % (label, e)))

    ### Download packages from the packagelist
    signal.signal(signal.SIGINT, signal.SIG_DFL)
    for pkg in package_list:
        ### FIXME: Check if not already on ISO-file or repository as well
        filename = '%s-%s-%s.%s.rpm' % (pkg[0], pkg[1], pkg[2], pkg[4])
        filesize = int(pkg[5])

        ### Filter packagelist
        if op.filter and not fnmatch.fnmatch(filename, op.filter):
            info(4, 'Packages %s excluded by filter' % filename)
            continue

        ### List only files if requested
        if op.list:
            info(0, filename)
            continue

        ### If file (or symlink target) exists
        if os.path.isfile(os.path.join(path, filename)):
            stat = os.stat(os.path.join(path, filename))
            if stat.st_size == filesize:
                info(3, 'File %s is already in %s' % (filename, path))
                continue
            else:
                info(2, 'File %s has wrong size (found: %s, expected: %s), refetching.' % (filename, stat.st_size, filesize))
                remove(os.path.join(path, filename))

        ### If symlink target does not exist, remove symlink
        elif os.path.islink(os.path.join(path, filename)):
            remove(os.path.join(path, filename))

        if op.dryrun:
            info(1, 'Not downloading package %s' % filename)
            continue

        try:
            if op.verbose <= 1:
                rpcServer.doCall(repos.getPackage, pkg, None, None)
            else:
                rpcServer.doCall(repos.getPackage, pkg, wrapperUtils.printPkg, wrapperUtils.printRetrieveHash)
        except rpclib.Fault, f:
            error(0, 'rpcError: Error getting package %s from %s. %s' % (filename, label, f.faultString))
        except TypeError, e:
            error(0, 'TypeError: Error downloading package %s from %s. Skipping.\n%s' % (filename, label, e))
        except up2dateErrors.CommunicationError, e:
            error(0, 'CommunicationError: Error downloading package %s from %s. Skipping.\n%s' % (filename, label, e))

        if op.source:
            try:
                hdr, call_type = rpcServer.doCall(repos.getHeader, pkg)
                srcrpm = hdr['sourcerpm']
                if op.verbose <= 1:
                    rpcServer.doCall(repos.getPackageSource, channel, srcrpm, None, None)
                else:
                    rpcServer.doCall(repos.getPackageSource, channel, srcrpm, wrapperUtils.printPkg, wrapperUtils.printRetrieveHash)
            except rpclib.Fault, f:
                error(0, 'rpcError: Error getting package %s from %s. %s' % (filename, label, f.faultString))
            except TypeError, e:
                error(0, 'TypeError: Error downloading package %s from %s. Skipping.\n%s' % (filename, label, e))
            except up2dateErrors.CommunicationError, e:
                error(0, 'CommunicationError: Error downloading package %s from %s. Skipping.\n%s' % (filename, label, e))

    ### Remove packages on the receiver side that are not on the sender side
    if op.cleanup:

        ### Collect receiver side
        receiver = Set()
        for file in glob.glob(os.path.join(path, '*.rpm')):
            if os.path.exists(file):
                filename = os.path.basename(file)
                filesize = os.stat(file).st_size
                receiver.add( (filename, filesize) )
        receiver.sort()

        ### Collect sender side
        sender = Set()
        for pkg in package_list:
            filename = '%s-%s-%s.%s.rpm' % (pkg[0], pkg[1], pkg[2], pkg[4])
            filesize = int(pkg[5])
            sender.add( (filename, filesize) )
        sender.sort()

        ### Remove difference between receiver and sender
        cleanse = receiver.difference(sender)
        for filename, filesize in cleanse.list:
            info(3, 'Cleaning up obsolete file %s (%d kiB)' % (filename, filesize))
            remove(os.path.join(path, filename))

def main():
    try:
        mirrorrhn(op.uri, op.destination)
    except Exception, e:
        die(1, e)

### Unbuffered sys.stdout
sys.stdout = os.fdopen(1, 'w', 0)
sys.stderr = os.fdopen(2, 'w', 0)

### Workaround for python <= 2.2.1
try:
     True, False
except NameError:
     True = 1
     False = 0

### Main entrance
if __name__ == '__main__':
    exitcode = 0

    op = Options(sys.argv[1:])
    try:
        main()
    except KeyboardInterrupt, e:
        die(6, 'Exiting on user request')
    sys.exit(exitcode)

# vim:ts=4:sw=4:et