-
Notifications
You must be signed in to change notification settings - Fork 2k
/
controller.py
90 lines (73 loc) · 3.36 KB
/
controller.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
from logging import getLogger
import urlparse
import requests
import pylons.config as config
import ckan.logic as logic
import ckan.lib.base as base
from ckan.common import _
import ckan.plugins.toolkit as toolkit
log = getLogger(__name__)
MAX_FILE_SIZE = toolkit.asint(
config.get('ckan.resource_proxy.max_file_size', 1024 * 1024))
CHUNK_SIZE = 512
def proxy_resource(context, data_dict):
''' Chunked proxy for resources. To make sure that the file is not too
large, first, we try to get the content length from the headers.
If the headers to not contain a content length (if it is a chinked
response), we only transfer as long as the transferred data is less
than the maximum file size. '''
resource_id = data_dict['resource_id']
log.info('Proxify resource {id}'.format(id=resource_id))
try:
resource = logic.get_action('resource_show')(context, {'id':
resource_id})
except logic.NotFound:
base.abort(404, _('Resource not found'))
url = resource['url']
parts = urlparse.urlsplit(url)
if not parts.scheme or not parts.netloc:
base.abort(409, detail='Invalid URL.')
try:
# first we try a HEAD request which may not be supported
did_get = False
r = requests.head(url)
# Servers can refuse HEAD requests. 405 is the appropriate response,
# but 400 with the invalid method mentioned in the text, or a 403
# (forbidden) status is also possible (#2412, #2530)
if r.status_code in (400, 403, 405):
r = requests.get(url, stream=True)
did_get = True
r.raise_for_status()
cl = r.headers.get('content-length')
if cl and int(cl) > MAX_FILE_SIZE:
base.abort(409, '''Content is too large to be proxied. Allowed
file size: {allowed}, Content-Length: {actual}.'''.format(
allowed=MAX_FILE_SIZE, actual=cl))
if not did_get:
r = requests.get(url, stream=True)
base.response.content_type = r.headers['content-type']
base.response.charset = r.encoding
length = 0
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
base.response.body_file.write(chunk)
length += len(chunk)
if length >= MAX_FILE_SIZE:
base.abort(409, headers={'content-encoding': ''},
detail='Content is too large to be proxied.')
except requests.exceptions.HTTPError, error:
details = 'Could not proxy resource. Server responded with %s %s' % (
error.response.status_code, error.response.reason)
base.abort(409, detail=details)
except requests.exceptions.ConnectionError, error:
details = '''Could not proxy resource because a
connection error occurred. %s''' % error
base.abort(502, detail=details)
except requests.exceptions.Timeout, error:
details = 'Could not proxy resource because the connection timed out.'
base.abort(504, detail=details)
class ProxyController(base.BaseController):
def proxy_resource(self, resource_id):
data_dict = {'resource_id': resource_id}
context = {'model': base.model, 'session': base.model.Session,
'user': base.c.user or base.c.author}
return proxy_resource(context, data_dict)