-
Notifications
You must be signed in to change notification settings - Fork 2k
/
controller.py
80 lines (66 loc) · 2.3 KB
/
controller.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
# encoding: utf-8
import StringIO
import pylons
from ckan.plugins.toolkit import (
Invalid,
ObjectNotFound,
get_action,
get_validator,
_,
request,
response,
BaseController,
abort,
)
from ckanext.datastore.writer import (
csv_writer,
tsv_writer,
)
int_validator = get_validator('int_validator')
boolean_validator = get_validator('boolean_validator')
DUMP_FORMATS = 'csv', 'tsv'
PAGINATE_BY = 10000
class DatastoreController(BaseController):
def dump(self, resource_id):
try:
offset = int_validator(request.GET.get('offset', 0), {})
except Invalid as e:
abort(400, u'offset: ' + e.error)
try:
limit = int_validator(request.GET.get('limit'), {})
except Invalid as e:
abort(400, u'limit: ' + e.error)
bom = boolean_validator(request.GET.get('bom'), {})
fmt = request.GET.get('format', 'csv')
def start_writer(columns):
if fmt == 'csv':
return csv_writer(response, columns, resource_id, bom)
if fmt == 'tsv':
return tsv_writer(response, columns, resource_id, bom)
abort(400, _(
u'format: must be one of %s') % u', '.join(DUMP_FORMATS))
def result_page(offset, limit):
try:
return get_action('datastore_search')(None, {
'resource_id': resource_id,
'limit':
PAGINATE_BY if limit is None
else min(PAGINATE_BY, limit),
'offset': offset,
})
except ObjectNotFound:
abort(404, _('DataStore resource not found'))
result = result_page(offset, limit)
columns = [x['id'] for x in result['fields']]
with start_writer(columns) as wr:
while True:
if limit is not None and limit <= 0:
break
for record in result['records']:
wr.writerow([record[column] for column in columns])
if len(result['records']) < PAGINATE_BY:
break
offset += PAGINATE_BY
if limit is not None:
limit -= PAGINATE_BY
result = result_page(offset, limit)