Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
tree: c4d2e8336a
Fetching contributors…

Cannot retrieve contributors at this time

executable file 256 lines (221 sloc) 8.775 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
manage_pkgdb.py
~~~~~~~~~~~~~~~

a simple script to update a Git repository with a package database.

:copyright: (c) 2010 by Rafael Goncalves Martins
:license: GPL-2, see LICENSE for more details.
"""

from __future__ import print_function

import datetime
import feedparser
import optparse
import os
import re
import shutil
import subprocess
import sys
import tarfile
import time
import urllib

from contextlib import closing

current_dir = os.path.dirname(os.path.realpath(__file__))
if os.path.exists(os.path.join(current_dir, '..', 'g_octave')):
    sys.path.insert(0, os.path.join(current_dir, '..'))

from g_octave import description_tree

class Git:

    def __init__(self, repo):
        self._repo = repo

    def __call__(self, args):
        args = args[:]
        args.insert(0, 'git')
        return subprocess.call(args, cwd=self._repo)

re_tarball = re.compile(r'(([^/]+)-([0-9.]+)\.tar\.gz)$')

class SfUpdates:

    # feed url from 'http://sourceforge.net/projects/octave/files/Octave%20Forge%20Packages/Individual%20Package%20Releases/'
    feed_url = u'http://sourceforge.net/api/file/index/project-id/2888/mtime/desc/rss?path=%2FOctave%20Forge%20Packages%2FIndividual%20Package%20Releases'

    svnroot_url = u'http://svn.code.sf.net/p/octave/code/trunk/octave-forge'
    categories = [u'main', u'extra', u'language', u'nonfree']

    _timestamp = None

    def __init__(self, local_dir, repo_dir):
        os.environ['GOCTAVE_DB'] = repo_dir
        self._local_dir = local_dir
        self._repo_dir = repo_dir
        self.feed = feedparser.parse(self.feed_url)
        if self.feed.bozo == 1:
            raise self.feed.bozo_exception
        self.entries = self.feed.entries

    def _save_timestamp(self):
        try:
            with open(os.path.join(self._repo_dir, 'timestamp'), 'w') as fp:
                fp.write(str(self._timestamp))
        except:
            pass

    def _load_timestamp(self):
        try:
            with open(os.path.join(self._repo_dir, 'timestamp')) as fp:
                return int(fp.read().strip())
        except:
            return 0

    def remote_files(self):
        # tarball name: {
        # name,
        # version,
        # download url
        # }
        entries = {}
        timestamp = self._load_timestamp()
        if self._timestamp is None:
            self._timestamp = timestamp
        for entry in self.entries:
            entry_timestamp = int(time.mktime(entry.updated_parsed))
            if entry_timestamp <= timestamp:
                break
            if entry_timestamp > self._timestamp:
                self._timestamp = entry_timestamp
            tarball = re_tarball.search(entry.summary)
            if tarball is not None:
                entries[tarball.group(1)] = {
                    'name': tarball.group(2),
                    'version': tarball.group(3),
                    'url': entry.link,
                }
        return entries

    def local_files(self):
        # tarball name: {
        # name,
        # version,
        # category
        # }
        db = description_tree.DescriptionTree()
        entries = {}
        for category in db.pkg_list:
            for pkg in db.pkg_list[category]:
                entries['%s-%s.tar.gz' % (pkg['name'], pkg['version'])] = {
                    'name': pkg['name'],
                    'version': pkg['version'],
                    'category': unicode(db.categories[pkg['name']]),
                }
        return entries

    def guess_category(self, pkgname):
        for category in self.categories:
            f = urllib.urlopen(self.svnroot_url + '/' + category + '/' + pkgname + '/DESCRIPTION')
            if f.getcode() == 200:
                return category

    def check_updates(self):
        local_files = self.local_files()
        remote_files = self.remote_files()
        updates = {}
        for remote in remote_files:
            print('update found: %s; ' % remote, end='')
            sys.stdout.flush()
            updates[remote] = remote_files[remote]
            category = self.guess_category(remote_files[remote]['name'])
            if category is None:
                remote_name = remote_files[remote]['name'].lower()
                for local in local_files:
                    local_name = local_files[local]['name'].lower()
                    if remote_name == local_name:
                        category = local_files[local]['category']
                        break
            remote_files[remote]['category'] = category
            print('category: %s' % category)
            if self.download(remote, remote_files[remote]) != os.EX_OK:
                raise RuntimeError('Failed to download: %s' % remote)
        return updates

    def download(self, tarball_name, entry):
        cat_dir = os.path.join(self._local_dir, entry['category'])
        file_path = os.path.join(cat_dir, tarball_name)
        if not os.path.exists(cat_dir):
            os.makedirs(cat_dir, 0o755)
        if not os.path.exists(file_path):
            return subprocess.call([
                'wget',
                '--continue',
                '--output-document', file_path,
                entry['url']
            ])
        return os.EX_OK

    def update_package_database(self, local_files, db_dir):
        if not os.path.exists(db_dir):
            os.makedirs(db_dir)
        for tarball_name in local_files:
            entry = local_files[tarball_name]
            description = os.path.join(
                db_dir,
                'octave-forge',
                entry['category'],
                entry['name'],
                '%s-%s.DESCRIPTION' % (entry['name'], entry['version'])
            )
            if not os.path.exists(description):
                if not os.path.exists(os.path.dirname(description)):
                    os.makedirs(os.path.dirname(description))
                tarball = os.path.join(
                    self._local_dir,
                    entry['category'],
                    tarball_name
                )
                with closing(tarfile.open(tarball, 'r:gz')) as src_tar:
                    to_extract = None
                    for f in src_tar:
                        if f.name.endswith('DESCRIPTION'):
                            to_extract = f
                            break
                    if f is None:
                        print('DESCRIPTION file not found: %s', tarball_name, file=sys.stderr)
                        continue
                    with closing(src_tar.extractfile(f)) as fp_tar:
                        with open(description, 'w') as fp:
                            shutil.copyfileobj(fp_tar, fp)
        self._save_timestamp()


def main(argv):

    parser = optparse.OptionParser(
        usage = '%prog [options] <sources directory> <package database directory>',
        version = 'see g-octave --version',
        description = 'a simple script to update a Git repository with a package database.'
    )

    parser.add_option(
        '-c', '--commit',
        action = 'store_true',
        dest = 'commit',
        default = False,
        help = 'commit the changes to the Git repository'
    )

    parser.add_option(
        '-p', '--push',
        action = 'store_true',
        dest = 'push',
        default = False,
        help = 'push the changes to the remote Git repository'
    )

    options, args = parser.parse_args(argv[1:])

    if len(args) != 2:
        print(
            'You need to provide 2 arguments:\n'
            '- the directory where you store the source tarballs\n'
            '- the directory where lives your package database (the git repository)',
            file=sys.stderr
        )
        return os.EX_USAGE

    print('* Fetching and parsing the Octave-Forge RSS feed ...')
    sf = SfUpdates(args[0], args[1])
    print('* Looking for updates ...')
    remote_files = sf.check_updates()
    print('* Trying to update the package database ...')
    sf.update_package_database(remote_files, args[1])
    if options.commit or options.push:
        print('* Trying to commit the changes to the package database Git repository ...')
        git = Git(args[1])
        date = datetime.datetime.utcnow().strftime('%H:%M:%S %Y/%m/%d')
        if git(['add', '.']) == os.EX_OK and \
           git(['commit', '-m', 'Package database update - %s' % date]) == os.EX_OK:
            if options.push:
                print('Trying to push the changes to the remote Git repository ...')
                return git(['push'])
            return os.EX_OK
        return os.EX_SOFTWARE
    return os.EX_OK


if __name__ == '__main__':
    sys.exit(main(sys.argv))
Something went wrong with that request. Please try again.