Feature/autoimport

extrapypi/commons/packages.py

@@ -97,9 +97,9 @@ def create_release(data, config, files):
                            .first()
     if release is None:
         release = Release(
-            description=data['description'],
-            download_url=data['download_url'],
-            home_page=data['home_page'],
+            description=data.get('description', 'UNKNOWN'),
+            download_url=data.get('download_url', 'UNKNOW'),
+            home_page=data.get('home_page', 'UNKNOWN'),
             version=data['version'],
             keywords=data.get('keywords'),
             md5_digest=data['md5_digest'],
@@ -115,3 +115,51 @@ def create_release(data, config, files):
         db.session.rollback()
         raise
     return release
+
+
+def create_release_from_source(metadata, user):
+    """Create a new release from a raw file. Used for import of existing packages into database
+
+    .. warning::
+
+        This function does not check any permissions since it's never called from web ui
+
+    If a release already exists, it does nothing
+
+    :param dict metadata: metadata of the package
+    :param extrapypi.models.User user: user to use as maintainer
+    """
+    try:
+        package = Package.query.filter_by(name=metadata['name']).one()
+    except NoResultFound:
+        package = Package(
+            name=metadata['name'],
+            summary=metadata.get('summary', 'UNKNOWN')
+        )
+        db.session.add(package)
+
+    release = Release.query.filter_by(version=metadata['version'], package=package)\
+                           .first()
+
+    if release is None:
+        release = Release(
+            description=metadata.get('description', 'UNKNOWN'),
+            download_url=metadata.get('download_url', 'UNKNOW'),
+            home_page=metadata.get('home_page', 'UNKNOWN'),
+            version=metadata['version'],
+            keywords=metadata.get('keywords'),
+            md5_digest=metadata['md5_digest'],
+            package=package
+        )
+        db.session.add(release)
+
+    if user not in package.maintainers:
+        package.maintainers.append(user)
+
+    try:
+        db.session.commit()
+    except Exception:
+        db.session.rollback()
+        raise
+
+    return release

extrapypi/dashboard/views.py

@@ -102,7 +102,7 @@ def package(package):
     release = p.latest_release
     store = get_store(app.config['STORAGE'], app.config['STORAGE_PARAMS'])
     files = store.get_files(p, release) or []
-    releases = [r for r in p.releases if r != release]
+    releases = [r for r in p.sorted_releases if r != release]
     return render_template("dashboard/package_detail.html",
                            release=release,
                            files=files,
@@ -125,7 +125,7 @@ def release(package, release_id):
 
     store = get_store(app.config['STORAGE'], app.config['STORAGE_PARAMS'])
     files = store.get_files(package, release) or []
-    releases = [r for r in package.releases if r != release]
+    releases = [r for r in package.sorted_releases if r != release]
     return render_template("dashboard/package_detail.html",
                            release=release,
                            files=files,

extrapypi/manage.py

@@ -77,7 +77,7 @@ def init_config(filename):
 def create_user(username, password, email, role):
     """Create a new user. Default role is admin
     """
-    from extrapypi.extrapypi import db
+    from extrapypi.extensions import db
     from extrapypi import models
     pwd = custom_app_context.hash(password)
     user = models.User(
@@ -91,10 +91,31 @@ def create_user(username, password, email, role):
         db.session.commit()
         click.echo("User %s created" % username)
     except Exception:
-        click.echo("Cannot create user %s" % username)
+        click.echo("Cannot create user %s" % username, err=True)
         db.rollback()
 
 
+@cli.command("import")
+@click.option("--user", default="admin")
+def reimport(user):
+    from extrapypi import models
+    from flask import current_app as app
+    from extrapypi.commons.packages import get_store, create_release_from_source
+
+    user_obj = models.User.query.filter_by(username=user).first()
+    if user_obj is None:
+        click.echo("Unknow user %s" % user, err=True)
+
+    store = get_store(
+        app.config['STORAGE'],
+        app.config['STORAGE_PARAMS']
+    )
+
+    for package, metadata in store.get_releases_metadata():
+        create_release_from_source(metadata, user_obj)
+        click.echo("created %s release for % s" % (metadata['version'], package))
+
+
 @cli.command()
 def db():
     """Flask-migrate commands"""

extrapypi/simple/views.py

@@ -59,15 +59,16 @@ def simple():
 def package_view(package):
     """List all files avaible for a package
     """
-    store = get_store(
-        current_app.config['STORAGE'],
-        current_app.config['STORAGE_PARAMS']
-    )
     try:
         package_obj = Package.query.filter_by(name=package).one()
     except NoResultFound:
         abort(404, "package %s does not exists" % package)
 
+    store = get_store(
+        current_app.config['STORAGE'],
+        current_app.config['STORAGE_PARAMS']
+    )
+
     files = store.get_files(package_obj)
     return render_template("simple/package.html",
                            files=files, package=package_obj)

extrapypi/storage/base.py

@@ -30,6 +30,14 @@ def __init__(self, **kwargs):
         for key, value in six.iteritems(kwargs):
             setattr(self, key, value)
 
+    def get_releases_metadata(self):
+        """Must return an iterable of tuples containing name of the package and release metadata
+
+        :return: list of all distributions contained in storage
+        :rtype: iterable
+        """
+        raise NotImplementedError()
+
     def delete_package(self, package):
         """Must delete an entire package
 

extrapypi/storage/local.py

@@ -7,7 +7,10 @@
 """
 import os
 import re
+import io
 import shutil
+import pkginfo
+from hashlib import md5
 
 from .base import BaseStorage
 
@@ -20,6 +23,32 @@ def __init__(self, packages_root=None):
             raise RuntimeError("Cannot use LocalStorage without PACKAGES_ROOT set")
         self.packages_root = packages_root
 
+    def _get_metadata(self, release):
+        try:
+            metadata = pkginfo.get_metadata(release).__dict__
+        except Exception:  # bad archive
+            metadata = {}
+
+        md5_hash = md5()
+
+        with open(release, 'rb') as fp:
+            for content in iter(lambda: fp.read(io.DEFAULT_BUFFER_SIZE), b''):
+                md5_hash.update(content)
+
+        metadata.update({'md5_digest': md5_hash.hexdigest()})
+        return metadata
+
+    def get_releases_metadata(self):
+        """List all releases metadata from PACKAGES_ROOT
+
+        :return: generator
+        :rtype: list
+        """
+        for root, dirs, files in os.walk(self.packages_root):
+            for f in files:
+                path = os.path.join(root, f)
+                yield (os.path.basename(path), self._get_metadata(path))
+
     def delete_package(self, package):
         """Delete entire package directory
         """
@@ -41,7 +70,7 @@ def delete_release(self, package, version):
             return False
 
         files = os.listdir(path)
-        regex = '{}-(?P<version>[0-9\.]*)[\.-].*'.format(package.name)
+        regex = '.*-(?P<version>[0-9\.]*)[\.-].*'
         r = re.compile(regex)
         files = filter(
             lambda f: r.match(f) and r.match(f).group('version') == version,
@@ -93,7 +122,7 @@ def get_files(self, package, release=None):
 
         files = os.listdir(path)
         if release is not None:
-            regex = '{}-(?P<version>[0-9\.]*)[\.-].*'.format(package.name)
+            regex = '.*-(?P<version>[0-9\.]*)[\.-].*'.format(package.name)
             r = re.compile(regex)
             v = release.version
             files = filter(

extrapypi/templates/dashboard/package_detail.html

@@ -46,6 +46,7 @@ <h1>{{release.package.name}} {{release.version}}</h1>
     <h3>Other releases</h3>
     {% for r in releases %}
     <a href="{{url_for('dashboard.release', package=release.package.name, release_id=r.id)}}">{{r.version}}</a>
+    <br/>
     {% endfor %}
   </div>
 </div>

requirements.txt

@@ -7,3 +7,4 @@ Flask-Principal
 passlib
 blinker
 docutils
+pkginfo

tests/test_commons.py

@@ -1,12 +1,14 @@
 import os
 import pytest
+import pkginfo
 from flask import Flask
 from flask_principal import Permission
 
+from extrapypi.extensions import db
 from extrapypi.app import configure_app
 from extrapypi.commons.filters import tohtml
 from extrapypi.commons.login import user_loader
-from extrapypi.commons.packages import get_store, create_package
+from extrapypi.commons.packages import get_store, create_package, create_release_from_source
 
 
 def test_get_store_errors():
@@ -22,6 +24,36 @@ def pass_permission(obj):
         create_package("test", "test", badstore)
 
 
+def test_create_release_from_source(admin_user, packages, tmpdir, monkeypatch):
+    metadata = {
+        'version': '0.1',
+        'description': 'test',
+        'name': 'unknow-package',
+        'summary': 'test',
+        'md5_digest': 'test'
+    }
+
+    # unknown package
+    r = create_release_from_source(metadata, admin_user)
+    assert r.version == '0.1'
+    assert r.description == 'test'
+    assert r.download_url == 'UNKNOW'
+
+    # known package
+    metadata['name'] = packages[0].name
+    r = create_release_from_source(metadata, admin_user)
+
+    # db exception
+    def raise_db():
+        raise Exception()
+
+    monkeypatch.setattr(db.session, 'commit', raise_db)
+    with pytest.raises(Exception):
+        r = create_release_from_source(metadata, admin_user)
+
+    monkeypatch.undo()
+
+
 def test_tohtml():
     assert tohtml("*test*") == "<p><em>test</em></p>\n"
 

tests/test_storage.py

@@ -31,6 +31,9 @@ def test_base_storage():
     with pytest.raises(NotImplementedError):
         bs.delete_package(None)
 
+    with pytest.raises(NotImplementedError):
+        bs.get_releases_metadata()
+
 
 def test_local_storage(app, db, tmpdir, releases, werkzeug_file):
     """Test local storage"""
@@ -49,7 +52,7 @@ def test_local_storage(app, db, tmpdir, releases, werkzeug_file):
     assert len(ls.get_files(package)) == 1
     assert ls.get_files(package)[0] == package.name + "-0.1.tar.gz"
     assert ls.create_release(package, werkzeug_file) is True
-    assert len(ls.get_files(package, package.releases[0])) == 1
+    assert len(ls.get_files(package, package.releases[0])) == 2
 
     f = ls.get_file(package, package.name + "-0.1.tar.gz")
     with open(f, 'r') as f:
@@ -58,6 +61,12 @@ def test_local_storage(app, db, tmpdir, releases, werkzeug_file):
     assert ls.delete_release(package, '0.1') is True
     assert ls.delete_package(package) is True
 
+    # test get releases metadata
+    metadata = list(ls.get_releases_metadata())
+    for path, meta in metadata:
+        assert 'test' in path
+        assert 'md5_digest' in meta
+
     # test with bad package
     package.name = "baddir"
     ls = LocalStorage(packages_root="bad/dir/location")