Implement uploading of packages.

This is mostly based on the code of chishop from
Ask Solem Hoel (https://github.com/ask)

AUTHORS

@@ -1,5 +1,6 @@
 Michael van Tellingen
 
 
-Contains code from Sentry:
-https://github.com/dcramer/sentry/contributors
+Contains code from:
+ - Sentry https://github.com/dcramer/sentry/contributors
+ - chishop https://github.com/ask/chishop/contributors

LICENSE

@@ -50,3 +50,34 @@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 POSSIBILITY OF SUCH DAMAGE.
 
+
+--
+
+Copyright (c) 2009, Ask Solem
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+Neither the name of Ask Solem nor the names of its contributors may be used
+to endorse or promote products derived from this software without specific
+prior written permission. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+

localshop/packages/forms.py

@@ -0,0 +1,27 @@
+from django import forms
+
+from localshop.packages import models
+
+
+class ReleaseForm(forms.ModelForm):
+    class Meta:
+        model = models.Release
+        exclude = ['classifiers', 'package']
+
+
+class ReleaseFileForm(forms.ModelForm):
+    class Meta:
+        model = models.ReleaseFile
+        exclude = ['size', 'release', 'filename']
+
+    def __init__(self, *args, **kwargs):
+        super(ReleaseFileForm, self).__init__(*args, **kwargs)
+        self.fields['pyversion'] = self.fields.pop('python_version')
+        self.fields['pyversion'].required = False
+
+    def save(self, commit=True):
+        obj = super(ReleaseFileForm, self).save(False)
+        obj.python_version = self.cleaned_data['pyversion'] or 'source'
+        if commit:
+            obj.save()
+        return obj

localshop/packages/models.py

@@ -2,10 +2,17 @@
 
 from django.db import models
 from django.core.urlresolvers import reverse
-
 from model_utils import Choices
 from model_utils.fields import AutoCreatedField, AutoLastModifiedField
-from model_utils.models import TimeStampedModel
+
+from localshop.packages.utils import OverwriteStorage
+
+
+class Classifier(models.Model):
+    name = models.CharField(max_length=255, unique=True)
+
+    def __unicode__(self):
+        return self.name
 
 
 class Package(models.Model):
@@ -15,7 +22,11 @@ class Package(models.Model):
 
     name = models.SlugField(max_length=200, unique=True)
 
-    uptime_timestamp = models.DateTimeField(null=True)
+    #: Indicate if this package is local (a private package)
+    is_local = models.BooleanField(default=False)
+
+    #: Timestamp when we last retrieved the metadata
+    update_timestamp = models.DateTimeField(null=True)
 
     def get_all_releases(self):
         result = {}
@@ -28,9 +39,29 @@ def get_all_releases(self):
 class Release(models.Model):
 
     created = AutoCreatedField()
+
     modified = AutoLastModifiedField()
 
+    author = models.CharField(max_length=128, blank=True)
+
+    author_email = models.CharField(max_length=255, blank=True)
+
+    classifiers = models.ManyToManyField(Classifier)
+
+    description = models.TextField(blank=True)
+
+    download_url = models.CharField(max_length=200, blank=True, null=True)
+
+    home_page = models.URLField(verify_exists=False, blank=True, null=True)
+
+    license = models.TextField(blank=True)
+
+    metadata_version = models.CharField(max_length=64, default=1.0)
+
     package = models.ForeignKey(Package, related_name="releases")
+
+    summary = models.TextField(blank=True)
+
     version = models.CharField(max_length=512)
 
 
@@ -64,20 +95,21 @@ class ReleaseFile(models.Model):
 
     size = models.IntegerField(null=True)
 
-    type = models.CharField(max_length=25, choices=TYPES)
+    filetype = models.CharField(max_length=25, choices=TYPES)
 
-    file = models.FileField(upload_to=release_file_upload_to, max_length=512)
+    distribution = models.FileField(upload_to=release_file_upload_to,
+        storage=OverwriteStorage(), max_length=512)
 
     filename = models.CharField(max_length=200, blank=True, null=True)
 
-    digest = models.CharField(max_length=512)
+    md5_digest = models.CharField(max_length=512)
 
     python_version = models.CharField(max_length=25)
 
     url = models.URLField(max_length=1024, blank=True)
 
     class Meta:
-        unique_together = ("release", "type", "python_version", "filename")
+        unique_together = ('release', 'filetype', 'python_version', 'filename')
 
     def get_absolute_url(self):
         url = reverse('packages:download', kwargs={

localshop/packages/pypi.py

@@ -52,10 +52,10 @@ def get_package_urls(name, package=None):
                     release=release, filename=info['filename'])
 
             release_file.python_version = info['python_version']
-            release_file.type = info['packagetype']
+            release_file.filetype = info['packagetype']
             release_file.url = info['url']
             release_file.size = info['size']
-            release_file.digest = info['md5_digest']
+            release_file.md5_digest = info['md5_digest']
             release_file.save()
 
     package.update_timestamp = datetime.datetime.utcnow()

localshop/packages/tasks.py

@@ -22,6 +22,6 @@ def download_file(pk):
 
     # Write the file to the django file field
     filename = os.path.basename(release_file.url)
-    release_file.file.save(filename, File(tmp_file))
+    release_file.distribution.save(filename, File(tmp_file))
     release_file.save()
     logging.info("Complete")

localshop/packages/tests/test_pypi.py

@@ -42,9 +42,9 @@ def side_effect(name, version):
 
         info = package.releases.get(version='0.1').files.all()[0]
         self.assertEqual(info.filename, 'localshop-0.1.tar.gz')
-        self.assertEqual(info.type, 'sdist')
+        self.assertEqual(info.filetype, 'sdist')
         self.assertEqual(info.python_version, 'source')
-        self.assertEqual(info.digest, '7ddf32e17a6ac5ce04a8ecbf782ca509')
+        self.assertEqual(info.md5_digest, '7ddf32e17a6ac5ce04a8ecbf782ca509')
         self.assertEqual(info.size, 23232)
         self.assertEqual(info.url, 'http://pypi.python.org/packages/source/r/'
             'localshop/localshop-0.1.tar.gz')

localshop/packages/tests/test_tasks.py

@@ -32,7 +32,7 @@ def test_download_file(self):
             tasks.download_file(release_file.pk)
 
         release_file = models.ReleaseFile.objects.get(pk=release_file.pk)
-        self.assertEqual(release_file.file.read(), 'test')
+        self.assertEqual(release_file.distribution.read(), 'test')
 
-        self.assertEqual(release_file.file.name,
+        self.assertEqual(release_file.distribution.name,
             'source/l/localshop/localshop-0.1.tar.gz')

localshop/packages/tests/test_utils.py

@@ -0,0 +1,97 @@
+from mock import Mock
+
+from django.test import TestCase
+from django.utils.datastructures import MultiValueDict
+
+from localshop.packages.utils import parse_distutils_request
+
+
+class TestParseDistutilsRequest(TestCase):
+    def test_register_post(self):
+        data = (
+            '\n----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="license"\n\n'
+            'BSD\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="name"\n\nlocalshop\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="metadata_version"\n\n'
+            '1.0\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="author"\n\n'
+            'Michael van Tellingen\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="home_page"\n\n'
+            'http://github.com/mvantellingen/localshop\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name=":action"\n\n'
+            'submit\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="download_url"\n\n'
+            'UNKNOWN\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="summary"\n\n'
+            'A private pypi server including auto-mirroring of pypi.\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="author_email"\n\n'
+            'michaelvantellingen@gmail.com\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="version"\n\n'
+            '0.1\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="platform"\n\n'
+            'UNKNOWN\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="classifiers"\n\n'
+            'Development Status :: 2 - Pre-Alpha\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="classifiers"\n\n'
+            'Framework :: Django\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="classifiers"\n\n'
+            'Intended Audience :: Developers\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="classifiers"\n\n'
+            'Intended Audience :: System Administrators\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="classifiers"\n\n'
+            'Operating System :: OS Independent\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="classifiers"\n\n'
+            'Topic :: Software Development\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254\n'
+            'Content-Disposition: form-data; name="description"\n\n'
+            'UNKNOWN\n'
+            '----------------GHSKFJDLGDS7543FJKLFHRE75642756743254--\n'
+        )
+        request = Mock()
+        request.raw_post_data = data
+        post, files = parse_distutils_request(request)
+
+        expected_post = MultiValueDict({
+            'name': ['localshop'],
+            'license': ['BSD'],
+            'author': ['Michael van Tellingen'],
+            'home_page': ['http://github.com/mvantellingen/localshop'],
+            ':action': ['submit'],
+            'download_url': [None],
+            'summary': [
+                'A private pypi server including auto-mirroring of pypi.'],
+            'author_email': ['michaelvantellingen@gmail.com'],
+            'metadata_version': ['1.0'],
+            'version': ['0.1'],
+            'platform': [None],
+            'classifiers': [
+                'Development Status :: 2 - Pre-Alpha',
+                'Framework :: Django',
+                'Intended Audience :: Developers',
+                'Intended Audience :: System Administrators',
+                'Operating System :: OS Independent',
+                'Topic :: Software Development'
+            ],
+            'description': [None]
+        })
+        expected_files = MultiValueDict()
+
+        self.assertEqual(post, expected_post)
+        self.assertEqual(files, expected_files)

localshop/packages/tests/test_views.py

@@ -0,0 +1,96 @@
+from django.test import TestCase
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.utils.datastructures import MultiValueDict
+
+from localshop.packages import models
+from localshop.packages import views
+
+
+class TestDistutilsViews(TestCase):
+
+    def test_register_new(self):
+        post = MultiValueDict({
+            'name': ['localshop'],
+            'license': ['BSD'],
+            'author': ['Michael van Tellingen'],
+            'home_page': ['http://github.com/mvantellingen/localshop'],
+            ':action': ['submit'],
+            'download_url': [None],
+            'summary': [
+                'A private pypi server including auto-mirroring of pypi.'],
+            'author_email': ['michaelvantellingen@gmail.com'],
+            'metadata_version': ['1.0'],
+            'version': ['0.1'],
+            'platform': [None],
+            'classifiers': [
+                'Development Status :: 2 - Pre-Alpha',
+                'Framework :: Django',
+                'Intended Audience :: Developers',
+                'Intended Audience :: System Administrators',
+                'Operating System :: OS Independent',
+                'Topic :: Software Development'
+            ],
+            'description': [None]
+        })
+        files = MultiValueDict()
+
+        response = views.handle_submit(post, files)
+        self.assertEqual(response.status_code, 200, response.content)
+
+        package = models.Package.objects.get(name='localshop')
+        self.assertEqual(package.releases.count(), 1)
+
+    def test_upload_new(self):
+        post = MultiValueDict({
+            'name': ['localshop'],
+            'license': ['BSD'],
+            'author': ['Michael van Tellingen'],
+            'home_page': ['http://github.com/mvantellingen/localshop'],
+            ':action': ['submit'],
+            'download_url': [None],
+            'summary': [
+                'A private pypi server including auto-mirroring of pypi.'],
+            'author_email': ['michaelvantellingen@gmail.com'],
+            'metadata_version': ['1.0'],
+            'version': ['0.1'],
+            'platform': [None],
+            'classifiers': [
+                'Development Status :: 2 - Pre-Alpha',
+                'Framework :: Django',
+                'Intended Audience :: Developers',
+                'Intended Audience :: System Administrators',
+                'Operating System :: OS Independent',
+                'Topic :: Software Development'
+            ],
+            'description': [None],
+
+            # Extra fields for upload
+            'pyversion': [''],
+            'filetype': ['sdist'],
+            'md5_digest': ['dc8f0311bb830ee96b8627f8335f2cb1'],
+        })
+        files = MultiValueDict({
+            'distribution': [
+                SimpleUploadedFile(
+                    'localshop-0.1.tar.gz', 'binary-test-data-here')
+            ]
+        })
+
+        response = views.handle_submit(post, files)
+        self.assertEqual(response.status_code, 200, response.content)
+
+        package = models.Package.objects.get(name='localshop')
+        self.assertEqual(package.releases.count(), 1)
+        self.assertTrue(package.is_local)
+
+        release = package.releases.all()[0]
+        self.assertEqual(release.files.count(), 1)
+
+        release_file = release.files.all()[0]
+        self.assertEqual(release_file.python_version, 'source')
+        self.assertEqual(release_file.filetype, 'sdist')
+        self.assertEqual(release_file.md5_digest,
+            'dc8f0311bb830ee96b8627f8335f2cb1')
+        self.assertEqual(release_file.filename, 'localshop-0.1.tar.gz')
+        self.assertEqual(release_file.distribution.read(),
+            'binary-test-data-here')