Permalink
Browse files

fixed the script manage_pkgdb.py and a test case

  • Loading branch information...
1 parent 83da922 commit c694ffa77b47d1d44fdb1a60c94a50ffd0a3c5a0 @rafaelmartins committed Oct 23, 2010
Showing with 101 additions and 84 deletions.
  1. +62 −56 contrib/manage_pkgdb.py
  2. +22 −18 g_octave/config.py
  3. +17 −10 tests/test_description.py
@@ -4,9 +4,9 @@
"""
manage_pkgdb.py
~~~~~~~~~~~~~~~
-
+
a simple script to update a Git repository with a package database.
-
+
:copyright: (c) 2010 by Rafael Goncalves Martins
:license: GPL-2, see LICENSE for more details.
"""
@@ -27,8 +27,14 @@
from contextlib import closing
+current_dir = os.path.dirname(os.path.realpath(__file__))
+if os.path.exists(os.path.join(current_dir, '..', 'g_octave')):
+ sys.path.insert(0, os.path.join(current_dir, '..'))
+
+from g_octave import description_tree
+
class Git:
-
+
def __init__(self, repo):
self._repo = repo
@@ -40,16 +46,17 @@ def __call__(self, args):
re_tarball = re.compile(r'(([^/]+)-([0-9.]+)\.tar\.gz)$')
class SfUpdates:
-
+
# feed url from 'http://sourceforge.net/projects/octave/files/Octave%20Forge%20Packages/Individual%20Package%20Releases/'
feed_url = u'http://sourceforge.net/api/file/index/project-id/2888/mtime/desc/rss?path=%2FOctave%20Forge%20Packages%2FIndividual%20Package%20Releases'
-
+
svnroot_url = u'https://octave.svn.sourceforge.net/svnroot/octave/trunk/octave-forge/'
categories = [u'main', u'extra', u'language', u'nonfree']
-
+
_timestamp = None
-
+
def __init__(self, local_dir, repo_dir):
+ os.environ['GOCTAVE_DB'] = repo_dir
self._local_dir = local_dir
self._repo_dir = repo_dir
self.feed = feedparser.parse(self.feed_url)
@@ -63,7 +70,7 @@ def _save_timestamp(self):
fp.write(str(self._timestamp))
except:
pass
-
+
def _load_timestamp(self):
try:
with open(os.path.join(self._repo_dir, 'timestamp')) as fp:
@@ -95,66 +102,65 @@ def remote_files(self):
'url': entry.link,
}
return entries
-
+
def local_files(self):
# tarball name: {
# name,
# version,
# category
# }
- if not os.path.isdir(self._local_dir):
- return {}
+ db = description_tree.DescriptionTree()
entries = {}
- for category in os.listdir(self._local_dir):
- for tarball in os.listdir(os.path.join(self._local_dir, category)):
- my_tarball = re_tarball.search(unicode(tarball))
- if my_tarball is not None:
- entries[my_tarball.group(1)] = {
- 'name': my_tarball.group(2),
- 'version': my_tarball.group(3),
- 'category': unicode(category),
- }
+ for category in db.pkg_list:
+ for pkg in db.pkg_list[category]:
+ entries['%s-%s.tar.gz' % (pkg['name'], pkg['version'])] = {
+ 'name': pkg['name'],
+ 'version': pkg['version'],
+ 'category': unicode(db.categories[pkg['name']]),
+ }
return entries
-
+
def guess_category(self, pkgname):
for category in self.categories:
f = urllib.urlopen(self.svnroot_url + '/' + category + '/' + pkgname + '/DESCRIPTION')
if f.getcode() == 200:
- return category
-
+ return category
+
def check_updates(self):
local_files = self.local_files()
remote_files = self.remote_files()
updates = {}
for remote in remote_files:
- if remote not in local_files:
- print('update found: %s; ' % remote, end='')
- updates[remote] = local_files[remote]
- category = self.guess_category(remote_files[remote]['name'])
- if category is None:
- remote_name = remote_files[remote]['name'].lower()
- for local in local_files:
- local_name = local_files[local]['name'].lower()
- if remote_name == local_name:
- category = local_files[local]['category']
- break
- remote_files[remote]['category'] = category
- print('category: %s' % category)
- if self.download(remote, remote_files[remote]) != os.EX_OK:
- raise RuntimeError('Failed to download: %s' % remote)
+ print('update found: %s; ' % remote, end='')
+ sys.stdout.flush()
+ updates[remote] = remote_files[remote]
+ category = self.guess_category(remote_files[remote]['name'])
+ if category is None:
+ remote_name = remote_files[remote]['name'].lower()
+ for local in local_files:
+ local_name = local_files[local]['name'].lower()
+ if remote_name == local_name:
+ category = local_files[local]['category']
+ break
+ remote_files[remote]['category'] = category
+ print('category: %s' % category)
+ if self.download(remote, remote_files[remote]) != os.EX_OK:
+ raise RuntimeError('Failed to download: %s' % remote)
return updates
-
+
def download(self, tarball_name, entry):
- return subprocess.call([
- 'wget',
- '--continue',
- '--output-document', os.path.join(
- self._local_dir,
- entry['category'],
- tarball_name,
- ),
- entry['url']
- ])
+ cat_dir = os.path.join(self._local_dir, entry['category'])
+ file_path = os.path.join(cat_dir, tarball_name)
+ if not os.path.exists(cat_dir):
+ os.makedirs(cat_dir, 0o755)
+ if not os.path.exists(file_path):
+ return subprocess.call([
+ 'wget',
+ '--continue',
+ '--output-document', file_path,
+ entry['url']
+ ])
+ return os.EX_OK
def update_package_database(self, local_files, db_dir):
if not os.path.exists(db_dir):
@@ -192,31 +198,31 @@ def update_package_database(self, local_files, db_dir):
def main(argv):
-
+
parser = optparse.OptionParser(
usage = '%prog [options] <sources directory> <package database directory>',
version = 'see g-octave --version',
description = 'a simple script to update a Git repository with a package database.'
)
-
+
parser.add_option(
'-c', '--commit',
action = 'store_true',
dest = 'commit',
default = False,
help = 'commit the changes to the Git repository'
)
-
+
parser.add_option(
'-p', '--push',
action = 'store_true',
dest = 'push',
default = False,
help = 'push the changes to the remote Git repository'
)
-
+
options, args = parser.parse_args(argv[1:])
-
+
if len(args) != 2:
print(
'You need to provide 2 arguments:\n'
@@ -225,7 +231,7 @@ def main(argv):
file=sys.stderr
)
return os.EX_USAGE
-
+
print('* Fetching and parsing the Octave-Forge RSS feed ...')
sf = SfUpdates(args[0], args[1])
print('* Looking for updates ...')
@@ -244,7 +250,7 @@ def main(argv):
return os.EX_OK
return os.EX_SOFTWARE
return os.EX_OK
-
+
if __name__ == '__main__':
sys.exit(main(sys.argv))
View
@@ -3,10 +3,10 @@
"""
config.py
~~~~~~~~~
-
+
This module implements a Python object to handle the configuration
of g-octave.
-
+
:copyright: (c) 2009-2010 by Rafael Goncalves Martins
:license: GPL-2, see LICENSE for more details.
"""
@@ -27,7 +27,7 @@
import ConfigParser as configparser
class Config(object):
-
+
_defaults = {
'db': '/var/cache/g-octave',
'overlay': '/var/lib/g-octave',
@@ -45,54 +45,58 @@ class Config(object):
_env_namespace = 'GOCTAVE_'
def __init__(self, fetch_phase=False, config_file=None, create_dirs=True):
-
+
# Config Parser
self._config = configparser.ConfigParser(self._defaults)
-
+
self._fetch_phase = fetch_phase
-
+
parsed_files = self._config.read([
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'..', 'etc', 'g-octave.cfg'
),
config_file or '/etc/g-octave.cfg',
])
-
+
if len(parsed_files) == 0:
raise ConfigException('Configuration file not found.')
-
+
_db = self._getattr('db')
_overlay = self._getattr('overlay')
-
+
for dir in [_db, _overlay]:
if not os.path.exists(dir) and create_dirs:
- os.makedirs(dir, 0o755)
-
+ try:
+ os.makedirs(dir, 0o755)
+ except:
+ # it's probably safe to ignore that
+ pass
+
self._cache = {}
self._info = {}
-
+
if not fetch_phase:
-
+
# JSON
json_file = os.path.join(_db, 'info.json')
with open(json_file) as fp:
self._info = json.load(fp)
-
+
def __getattr__(self, attr):
-
+
if attr in self._defaults:
return self._getattr(attr)
elif attr in self._info:
return self._info[attr]
else:
raise ConfigException('Invalid option: %s' % attr)
-
-
+
+
def _getattr(self, attr):
from_env = os.environ.get(self._env_namespace + attr.upper(), None)
if from_env is None:
return self._config.get(self._section_name, attr)
return from_env
-
+
@@ -4,28 +4,31 @@
"""
test_description.py
~~~~~~~~~~~~~~~~~~~
-
+
test suite for the *g_octave.description* module
-
+
:copyright: (c) 2010 by Rafael Goncalves Martins
:license: GPL-2, see LICENSE for more details.
"""
import os
import unittest
+import utils
from g_octave import description
class TestDescription(unittest.TestCase):
-
+
def setUp(self):
+ conf, self._config_file, self._tempdir = utils.create_env()
self.desc = description.Description(
os.path.join(
- os.path.dirname(os.path.abspath(__file__)), 'files', 'DESCRIPTION'
- )
+ os.path.dirname(os.path.abspath(__file__)), 'files', 'DESCRIPTION',
+ ),
+ conf = conf
)
-
+
def test_re_depends(self):
depends = [
('pkg', ('pkg', None, None)),
@@ -156,7 +159,7 @@ def test_re_depends(self):
(match.group(1), match.group(3), match.group(4)),
pkgtpl
)
-
+
def test_re_pkg_atom(self):
depends = [
('pkg-1', ('pkg', '1')),
@@ -182,20 +185,24 @@ def test_attributes(self):
self.assertEqual(self.desc.description, 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.')
self.assertEqual(self.desc.categories, 'Category1,Category2, Category3')
self.assertEqual(self.desc.url, 'http://example.org')
-
+
requirements = [
'>=g-octave/pkg1-4.3.2',
'<g-octave/pkg2-1.2.3',
'g-octave/pkg3'
]
- requirements.sort()
+ requirements.sort()
self.assertEqual(self.desc.systemrequirements, requirements)
self.assertEqual(self.desc.buildrequires, ['>g-octave/pkg4-1.0.0'])
-
+
self.assertEqual(self.desc.depends, ['>=sci-mathematics/octave-3.0.0'])
self.assertEqual(self.desc.autoload, 'NO')
self.assertEqual(self.desc.license, 'GPL version 3 or later')
+ def tearDown(self):
+ # removing the temp tree
+ utils.clean_env(self._config_file, self._tempdir)
+
def suite():
suite = unittest.TestSuite()

0 comments on commit c694ffa

Please sign in to comment.