[upload][m]: add support for uploading to local file storage as oppos…

…ed to remote storage.

* doc: also improve docs giving install instructions, more detail of config options etc
* setup.py: bump version to 0.5a

.hgignore

@@ -4,4 +4,5 @@ syntax: glob
 *.egg-info/*
 sandbox/*
 .DS_Store
+build/*
 

README.rst

@@ -10,17 +10,46 @@ This extension adds:
 It uses `OFS`_ to talk to the backing storage so can support anything that OFS
 supports including local filesytem, S3, Google Storage etc.
 
-.. _OFS: http://pypi.python.org/pypi/ofs/
+.. _OFS: http://packages.python.org/ofs/
+
+Installation
+============
+
+Install the extension::
+
+    # using pip (could use easy_install)
+    pip install ckanext-storage
+    # could install from source
+    # hg clone https://bitbucket.org/okfn/ckanext-storage
+    # cd ckanext-storage
+    # pip install -e .
+
+Note that for use of S3-like backends (S3, Google Storage etc) you will need boto (this is installed by default at the moment). For local filesystem backend you need to install pairtree (`pip install pairtree`).
 
 In your config you need something like::
 
    ckan.plugins = storage
-   # this is for google storage
+
+   ## OFS configuration
+   ## This is for google storage. Example for another backend is below
+   ## See OFS docs for full details
    ofs.impl = google
    ofs.gs_access_key_id = GOOGCABCDASDASD
    ofs.gs_secret_access_key = 134zsdfjkw4234addad
-   ckanext.storage.bucket = the bucket to use for uploading
-   ckanext.storage.max_content_length = [optional] maximum content size for uploads (defaults to 50Mb)
+   ## bucket to use in storage You *must* set this
+   ckanext.storage.bucket = ....
+
+   ## optional
+   ## maximum content size for uploads in bytes, defaults to 1Gb
+   # ckanext.storage.max_content_length = 1000000000
+   ## prefix for all keys. Useful because we may use a single bucket and want to
+   ## partition file uploads. Defaults to file/
+   # ckanext.storage.key_prefix = file/
+
+For local file storage you would replace ofs arguments with::
+
+   ofs.impl = pairtree
+   ofs.storage_dir = /my/path/to/storage/root/directory
 
 
 Upload Web Interface

ckanext/storage/__init__.py

@@ -30,7 +30,10 @@ def after_map(self, route_map):
         # upload page
         route_map.connect('storage_upload', '/storage/upload',
             controller='ckanext.storage.controller:StorageController',
-            action='index')
+            action='upload')
+        route_map.connect('storage_upload_handle', '/storage/upload_handle',
+            controller='ckanext.storage.controller:StorageController',
+            action='upload_handle')
         route_map.connect('storage_upload_success', '/storage/upload/success',
             controller='ckanext.storage.controller:StorageController',
             action='success')

ckanext/storage/controller.py

@@ -1,5 +1,6 @@
 import re
 from datetime import datetime
+from cgi import FieldStorage
 try:
     from cStringIO import StringIO
 except ImportError:
@@ -16,6 +17,7 @@
 from pylons import request, response
 from pylons.controllers.util import abort, redirect_to
 from pylons import config
+from paste.fileapp import FileApp
 
 from ckan.lib.base import BaseController, c, request, render, config, h, abort
 from ckan.lib.jsonp import jsonpify
@@ -251,15 +253,14 @@ class StorageController(BaseController):
     '''
     ofs = get_ofs()
 
-    def index(self):
-        label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
+    def _get_form_for_remote(self):
         # would be nice to use filename of file
         # problem is 'we' don't know this at this point and cannot add it to
         # success_action_redirect and hence cannnot display to user afterwards
         # + '/${filename}'
+        label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
         method = 'POST'
         authorize(method, BUCKET, label, c.userobj, self.ofs)
-
         content_length_range = int(
                 config.get('ckanext.storage.max_content_length',
                     50000000))
@@ -277,7 +278,7 @@ def index(self):
         c.data = self.ofs.conn.build_post_form_args(
             BUCKET,
             label,
-            expires_in=600,
+            expires_in=3600,
             max_content_length=content_length_range,
             success_action_redirect=success_action_redirect,
             acl=acl,
@@ -293,8 +294,43 @@ def index(self):
             if field['name'] == 'content-length-range':
                 del c.data['fields'][idx]
         c.data_json = json.dumps(c.data, indent=2)
+
+    def upload(self):
+        if storage_backend in ['google', 's3']:
+            self._get_form_for_remote()
+        else:
+            label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
+            c.data = {
+                'action': h.url_for('storage_upload_handle'),
+                'fields': [
+                    {
+                        'name': 'key',
+                        'value': label
+                    }
+                ]
+            }
         return render('ckanext/storage/index.html')
 
+    def upload_handle(self):
+        bucket_id = BUCKET
+        params = request.params
+        params = dict(params.items())
+        stream = params.get('file')
+        label = params.get('key')
+        authorize('POST', BUCKET, label, c.userobj, self.ofs)
+        if not label:
+            abort(400, "No label")
+        if not isinstance(stream, FieldStorage):
+            abort(400, "No file stream.")
+        del params['file']
+        params['filename-original'] = stream.filename
+        params['_owner'] = c.userobj.id
+        params['uploaded-by'] = c.userobj.id
+        self.ofs.put_stream(bucket_id, label, stream.file, params)
+        success_action_redirect = h.url_for('storage_upload_success', qualified=True,
+                bucket=BUCKET, label=label)
+        h.redirect_to(success_action_redirect)
+
     def success(self):
         h.flash_success('Upload successful')
         c.file_url = h.url_for('storage_file',
@@ -305,6 +341,15 @@ def success(self):
         return render('ckanext/storage/success.html')
 
     def file(self, label):
-        url = self.ofs.get_url(BUCKET, label)
-        h.redirect_to(url)
+        file_url = self.ofs.get_url(BUCKET, label)
+        if file_url.startswith("file://"):
+            metadata = self.ofs.get_metadata(BUCKET, label)
+            filepath = file_url[len("file://"):]
+            headers = {'Content-Disposition':'attachment; filename="%s"' %
+                    label,
+                       'Content-Type':metadata.get('_format', 'text/plain')}
+            fapp = FileApp(filepath, headers=None, **headers)
+            return fapp(request.environ, self.start_response)
+        else:
+            h.redirect_to(file_url)
 

ckanext/storage/templates/ckanext/storage/index.html

@@ -14,8 +14,8 @@
   <div py:match="content">
     <h1>Upload</h1>
 
-    <p>This upload form is valid for 10m. Please reload the page after 10m to
-    start again.</p>
+    <p>This upload form is valid for a limited time (usually 1h or so). If the
+    form expires please reload the page.</p>
 
     <form action="${c.data['action']}"
       enctype="multipart/form-data"

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 import sys, os
 
-version = '0.4'
+version = '0.5a'
 try:
     long_description = open('README.txt').read()
 except: