-
Notifications
You must be signed in to change notification settings - Fork 2k
/
storage.py
120 lines (97 loc) · 3.45 KB
/
storage.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
import os
import re
import urllib
import uuid
from datetime import datetime
from cgi import FieldStorage
from ofs import get_impl
from pylons import request, response
from pylons.controllers.util import abort, redirect_to
from pylons import config
from paste.fileapp import FileApp
from paste.deploy.converters import asbool
from ckan.lib.base import BaseController, c, request, render, config, h, abort
from ckan.lib.jsonp import jsonpify
import ckan.model as model
import ckan.logic as logic
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
try:
import json
except:
import simplejson as json
from logging import getLogger
log = getLogger(__name__)
BUCKET = config.get('ckan.storage.bucket', 'default')
key_prefix = config.get('ckan.storage.key_prefix', 'file/')
_eq_re = re.compile(r"^(.*)(=[0-9]*)$")
def fix_stupid_pylons_encoding(data):
"""
Fix an apparent encoding problem when calling request.body
TODO: Investigate whether this is fixed in later versions?
"""
if data.startswith("%") or data.startswith("+"):
data = urllib.unquote_plus(data)
m = _eq_re.match(data)
if m:
data = m.groups()[0]
return data
def create_pairtree_marker(folder):
""" Creates the pairtree marker for tests if it doesn't exist """
if not folder[:-1] == '/':
folder = folder + '/'
directory = os.path.dirname(folder)
if not os.path.exists(directory):
os.makedirs(directory)
target = os.path.join(directory, 'pairtree_version0_1')
if os.path.exists(target):
return
open(target, 'wb').close()
def get_ofs():
"""Return a configured instance of the appropriate OFS driver.
"""
storage_backend = config['ofs.impl']
kw = {}
for k, v in config.items():
if not k.startswith('ofs.') or k == 'ofs.impl':
continue
kw[k[4:]] = v
# Make sure we have created the marker file to avoid pairtree issues
if storage_backend == 'pairtree' and 'storage_dir' in kw:
create_pairtree_marker(kw['storage_dir'])
ofs = get_impl(storage_backend)(**kw)
return ofs
class StorageController(BaseController):
'''Upload to storage backend.
'''
_ofs_impl = None
@property
def ofs(self):
if not StorageController._ofs_impl:
StorageController._ofs_impl = get_ofs()
return StorageController._ofs_impl
def file(self, label):
exists = self.ofs.exists(BUCKET, label)
if not exists:
# handle erroneous trailing slash by redirecting to url w/o slash
if label.endswith('/'):
label = label[:-1]
# This may be best being cached_url until we have moved it into
# permanent storage
file_url = h.url_for('storage_file', label=label)
h.redirect_to(file_url)
else:
abort(404)
file_url = self.ofs.get_url(BUCKET, label)
if file_url.startswith("file://"):
metadata = self.ofs.get_metadata(BUCKET, label)
filepath = file_url[len("file://"):]
headers = {
# 'Content-Disposition':'attachment; filename="%s"' % label,
'Content-Type': metadata.get('_format', 'text/plain')}
fapp = FileApp(filepath, headers=None, **headers)
return fapp(request.environ, self.start_response)
else:
h.redirect_to(file_url.encode('ascii', 'ignore'))