-
Notifications
You must be signed in to change notification settings - Fork 2k
/
__init__.py
679 lines (530 loc) · 23 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
import functools
import logging
import re
import sys
import formencode.validators
import ckan.model as model
import ckan.new_authz as new_authz
import ckan.lib.navl.dictization_functions as df
import ckan.plugins as p
from ckan.common import _, c
log = logging.getLogger(__name__)
_validate = df.validate
class NameConflict(Exception):
pass
class AttributeDict(dict):
def __getattr__(self, name):
try:
return self[name]
except KeyError:
raise AttributeError('No such attribute %r' % name)
def __setattr__(self, name, value):
raise AttributeError(
'You cannot set attributes of this object directly'
)
class ActionError(Exception):
def __init__(self, extra_msg=None):
self.extra_msg = extra_msg
def __str__(self):
err_msgs = (super(ActionError, self).__str__(),
self.extra_msg)
return ' - '.join([str(err_msg) for err_msg in err_msgs if err_msg])
class NotFound(ActionError):
'''Exception raised by logic functions when a given object is not found.
For example :py:func:`~ckan.logic.action.get.package_show` raises
:py:exc:`~ckan.plugins.toolkit.ObjectNotFound` if no package with the
given ``id`` exists.
'''
pass
class NotAuthorized(ActionError):
'''Exception raised when the user is not authorized to call the action.
For example :py:func:`~ckan.logic.action.create.package_create` raises
:py:exc:`~ckan.plugins.toolkit.NotAuthorized` if the user is not authorized
to create packages.
'''
pass
class ValidationError(ActionError):
'''Exception raised by action functions when validating their given
``data_dict`` fails.
'''
def __init__(self, error_dict, error_summary=None, extra_msg=None):
if not isinstance(error_dict, dict):
error_dict = {'message': error_dict}
# tags errors are a mess so let's clean them up
if 'tags' in error_dict:
tag_errors = []
for error in error_dict['tags']:
try:
tag_errors.append(', '.join(error['name']))
except KeyError:
pass
error_dict['tags'] = tag_errors
self.error_dict = error_dict
self._error_summary = error_summary
self.extra_msg = extra_msg
@property
def error_summary(self):
''' autogenerate the summary if not supplied '''
def summarise(error_dict):
''' Do some i18n stuff on the error_dict keys '''
def prettify(field_name):
field_name = re.sub('(?<!\w)[Uu]rl(?!\w)', 'URL',
field_name.replace('_', ' ').capitalize())
return _(field_name.replace('_', ' '))
summary = {}
for key, error in error_dict.iteritems():
if key == 'resources':
summary[_('Resources')] = _('Package resource(s) invalid')
elif key == 'extras':
errors_extras = []
for item in error:
if (item.get('key')
and item['key'][0] not in errors_extras):
errors_extras.append(item.get('key')[0])
summary[_('Extras')] = ', '.join(errors_extras)
elif key == 'extras_validation':
summary[_('Extras')] = error[0]
elif key == 'tags':
summary[_('Tags')] = error[0]
else:
summary[_(prettify(key))] = error[0]
return summary
if self._error_summary:
return self._error_summary
return summarise(self.error_dict)
def __str__(self):
err_msgs = (super(ValidationError, self).__str__(),
self.error_dict)
return ' - '.join([str(err_msg) for err_msg in err_msgs if err_msg])
log = logging.getLogger(__name__)
def parse_params(params, ignore_keys=None):
'''Takes a dict and returns it with some values standardised.
This is done on a dict before calling tuplize_dict on it.
'''
parsed = {}
for key in params:
if ignore_keys and key in ignore_keys:
continue
value = params.getall(key)
# Blank values become ''
if not value:
value = ''
# A list with only one item is stripped of being a list
if len(value) == 1:
value = value[0]
parsed[key] = value
return parsed
def clean_dict(data_dict):
'''Takes a dict and if any of the values are lists of dicts,
the empty dicts are stripped from the lists (recursive).
e.g.
>>> clean_dict(
{'name': u'testgrp4',
'title': u'',
'description': u'',
'packages': [{'name': u'testpkg'}, {'name': u'testpkg'}],
'extras': [{'key': u'packages', 'value': u'["testpkg"]'},
{'key': u'', 'value': u''},
{'key': u'', 'value': u''}],
'state': u'active'}
{'name': u'testgrp4',
'title': u'',
'description': u'',
'packages': [{'name': u'testpkg'}, {'name': u'testpkg'}],
'extras': [{'key': u'packages', 'value': u'["testpkg"]'}],
'state': u'active'}
'''
for key, value in data_dict.items():
if not isinstance(value, list):
continue
for inner_dict in value[:]:
if isinstance(inner_dict, basestring):
break
if not any(inner_dict.values()):
value.remove(inner_dict)
else:
clean_dict(inner_dict)
return data_dict
def tuplize_dict(data_dict):
'''Takes a dict with keys of the form 'table__0__key' and converts them
to a tuple like ('table', 0, 'key').
Dict should be put through parse_dict before this function, to have
values standardized.
May raise a DataError if the format of the key is incorrect.
'''
tuplized_dict = {}
for key, value in data_dict.iteritems():
key_list = key.split('__')
for num, key in enumerate(key_list):
if num % 2 == 1:
try:
key_list[num] = int(key)
except ValueError:
raise df.DataError('Bad key')
tuplized_dict[tuple(key_list)] = value
return tuplized_dict
def untuplize_dict(tuplized_dict):
data_dict = {}
for key, value in tuplized_dict.iteritems():
new_key = '__'.join([str(item) for item in key])
data_dict[new_key] = value
return data_dict
def flatten_to_string_key(dict):
flattented = df.flatten_dict(dict)
return untuplize_dict(flattented)
def _prepopulate_context(context):
if context is None:
context = {}
context.setdefault('model', model)
context.setdefault('session', model.Session)
try:
context.setdefault('user', c.user or c.author)
except TypeError:
# c not registered
pass
return context
def check_access(action, context, data_dict=None):
'''Calls the authorization function for the provided action
This is the only function that should be called to determine whether a
user (or an anonymous request) is allowed to perform a particular action.
The function accepts a context object, which should contain a 'user' key
with the name of the user performing the action, and optionally a
dictionary with extra data to be passed to the authorization function.
For example::
check_access('package_update', context, data_dict)
If not already there, the function will add an `auth_user_obj` key to the
context object with the actual User object (in case it exists in the
database). This check is only performed once per context object.
Raise :py:exc:`~ckan.plugins.toolkit.NotAuthorized` if the user is not
authorized to call the named action function.
If the user *is* authorized to call the action, return ``True``.
:param action: the name of the action function, eg. ``'package_create'``
:type action: string
:param context:
:type context: dict
:param data_dict:
:type data_dict: dict
:raises: :py:exc:`~ckan.plugins.toolkit.NotAuthorized` if the user is not
authorized to call the named action
'''
# Auth Auditing. We remove this call from the __auth_audit stack to show
# we have called the auth function
try:
audit = context.get('__auth_audit', [])[-1]
except IndexError:
audit = ''
if audit and audit[0] == action:
context['__auth_audit'].pop()
user = context.get('user')
log.debug('check access - user %r, action %s' % (user, action))
if not 'auth_user_obj' in context:
context['auth_user_obj'] = None
if not context.get('ignore_auth'):
if not context.get('__auth_user_obj_checked'):
if context.get('user') and not context.get('auth_user_obj'):
context['auth_user_obj'] = model.User.by_name(context['user'])
context['__auth_user_obj_checked'] = True
context = _prepopulate_context(context)
logic_authorization = new_authz.is_authorized(action, context, data_dict)
if not logic_authorization['success']:
msg = logic_authorization.get('msg', '')
raise NotAuthorized(msg)
log.debug('Access OK.')
return True
_actions = {}
def clear_actions_cache():
_actions.clear()
def get_action(action):
'''Return the named :py:mod:`ckan.logic.action` function.
For example ``get_action('package_create')`` will normally return the
:py:func:`ckan.logic.action.create.package_create()` function.
For documentation of the available action functions, see
:ref:`api-reference`.
You should always use ``get_action()`` instead of importing an action
function directly, because :py:class:`~ckan.plugins.interfaces.IActions`
plugins can override action functions, causing ``get_action()`` to return a
plugin-provided function instead of the default one.
Usage::
import ckan.plugins.toolkit as toolkit
# Call the package_create action function:
toolkit.get_action('package_create')(context, data_dict)
As the context parameter passed to an action function is commonly::
context = {'model': ckan.model, 'session': ckan.model.Session,
'user': pylons.c.user or pylons.c.author}
an action function returned by ``get_action()`` will automatically add
these parameters to the context if they are not defined. This is
especially useful for plugins as they should not really be importing parts
of ckan eg :py:mod:`ckan.model` and as such do not have access to ``model``
or ``model.Session``.
If a ``context`` of ``None`` is passed to the action function then the
default context dict will be created.
:param action: name of the action function to return,
eg. ``'package_create'``
:type action: string
:returns: the named action function
:rtype: callable
'''
if _actions:
if not action in _actions:
raise KeyError("Action '%s' not found" % action)
return _actions.get(action)
# Otherwise look in all the plugins to resolve all possible
# First get the default ones in the ckan/logic/action directory
# Rather than writing them out in full will use __import__
# to load anything from ckan.logic.action that looks like it might
# be an action
for action_module_name in ['get', 'create', 'update', 'delete', 'patch']:
module_path = 'ckan.logic.action.' + action_module_name
module = __import__(module_path)
for part in module_path.split('.')[1:]:
module = getattr(module, part)
for k, v in module.__dict__.items():
if not k.startswith('_'):
# Only load functions from the action module or already
# replaced functions.
if (hasattr(v, '__call__')
and (v.__module__ == module_path
or hasattr(v, '__replaced'))):
_actions[k] = v
# Whitelist all actions defined in logic/action/get.py as
# being side-effect free.
if action_module_name == 'get' and \
not hasattr(v, 'side_effect_free'):
v.side_effect_free = True
# Then overwrite them with any specific ones in the plugins:
resolved_action_plugins = {}
fetched_actions = {}
for plugin in p.PluginImplementations(p.IActions):
for name, auth_function in plugin.get_actions().items():
if name in resolved_action_plugins:
raise NameConflict(
'The action %r is already implemented in %r' % (
name,
resolved_action_plugins[name]
)
)
log.debug('Action function {0} from plugin {1} was inserted'.format(name, plugin.name))
resolved_action_plugins[name] = plugin.name
# Extensions are exempted from the auth audit for now
# This needs to be resolved later
auth_function.auth_audit_exempt = True
fetched_actions[name] = auth_function
# Use the updated ones in preference to the originals.
_actions.update(fetched_actions)
# wrap the functions
for action_name, _action in _actions.items():
def make_wrapped(_action, action_name):
def wrapped(context=None, data_dict=None, **kw):
if kw:
log.critical('%s was passed extra keywords %r'
% (_action.__name__, kw))
context = _prepopulate_context(context)
# Auth Auditing
# store this action name in the auth audit so we can see if
# check access was called on the function we store the id of
# the action incase the action is wrapped inside an action
# of the same name. this happens in the datastore
context.setdefault('__auth_audit', [])
context['__auth_audit'].append((action_name, id(_action)))
# check_access(action_name, context, data_dict=None)
result = _action(context, data_dict, **kw)
try:
audit = context['__auth_audit'][-1]
if audit[0] == action_name and audit[1] == id(_action):
if action_name not in new_authz.auth_functions_list():
log.debug('No auth function for %s' % action_name)
elif not getattr(_action, 'auth_audit_exempt', False):
raise Exception(
'Action function {0} did not call its auth function'
.format(action_name))
# remove from audit stack
context['__auth_audit'].pop()
except IndexError:
pass
return result
return wrapped
# If we have been called multiple times for example during tests then
# we need to make sure that we do not rewrap the actions.
if hasattr(_action, '__replaced'):
_actions[action_name] = _action.__replaced
continue
fn = make_wrapped(_action, action_name)
# we need to mirror the docstring
fn.__doc__ = _action.__doc__
# we need to retain the side effect free behaviour
if getattr(_action, 'side_effect_free', False):
fn.side_effect_free = True
_actions[action_name] = fn
return _actions.get(action)
def get_or_bust(data_dict, keys):
'''Return the value(s) from the given data_dict for the given key(s).
Usage::
single_value = get_or_bust(data_dict, 'a_key')
value_1, value_2 = get_or_bust(data_dict, ['key1', 'key2'])
:param data_dict: the dictionary to return the values from
:type data_dict: dictionary
:param keys: the key(s) for the value(s) to return
:type keys: either a string or a list
:returns: a single value from the dict if a single key was given,
or a tuple of values if a list of keys was given
:raises: :py:exc:`ckan.logic.ValidationError` if one of the given keys is
not in the given dictionary
'''
if isinstance(keys, basestring):
keys = [keys]
import ckan.logic.schema as schema
schema = schema.create_schema_for_required_keys(keys)
data_dict, errors = _validate(data_dict, schema)
if errors:
raise ValidationError(errors)
# preserve original key order
values = [data_dict[key] for key in keys]
if len(values) == 1:
return values[0]
return tuple(values)
def validate(schema_func, can_skip_validator=False):
''' A decorator that validates an action function against a given schema
'''
def action_decorator(action):
@functools.wraps(action)
def wrapper(context, data_dict):
if can_skip_validator:
if context.get('skip_validation'):
return action(context, data_dict)
schema = context.get('schema', schema_func())
data_dict, errors = _validate(data_dict, schema, context)
if errors:
raise ValidationError(errors)
return action(context, data_dict)
return wrapper
return action_decorator
def side_effect_free(action):
'''A decorator that marks the given action function as side-effect-free.
Action functions decorated with this decorator can be called with an HTTP
GET request to the :doc:`Action API </api/index>`. Action functions that
don't have this decorator must be called with a POST request.
If your CKAN extension defines its own action functions using the
:py:class:`~ckan.plugins.interfaces.IActions` plugin interface, you can use
this decorator to make your actions available with GET requests instead of
just with POST requests.
Example::
import ckan.plugins.toolkit as toolkit
@toolkit.side_effect_free
def my_custom_action_function(context, data_dict):
...
(Then implement :py:class:`~ckan.plugins.interfaces.IActions` to register
your action function with CKAN.)
'''
@functools.wraps(action)
def wrapper(context, data_dict):
return action(context, data_dict)
wrapper.side_effect_free = True
return wrapper
def auth_sysadmins_check(action):
'''A decorator that prevents sysadmins from being automatically authorized
to call an action function.
Normally sysadmins are allowed to call any action function (for example
when they're using the :doc:`Action API </api/index>` or the web
interface), if the user is a sysadmin the action function's authorization
function will not even be called.
If an action function is decorated with this decorator, then its
authorization function will always be called, even if the user is a
sysadmin.
'''
@functools.wraps(action)
def wrapper(context, data_dict):
return action(context, data_dict)
wrapper.auth_sysadmins_check = True
return wrapper
def auth_audit_exempt(action):
''' Dirty hack to stop auth audit being done '''
@functools.wraps(action)
def wrapper(context, data_dict):
return action(context, data_dict)
wrapper.auth_audit_exempt = True
return wrapper
def auth_allow_anonymous_access(action):
''' Flag an auth function as not requiring a logged in user
This means that check_access won't automatically raise a NotAuthorized
exception if an authenticated user is not provided in the context. (The
auth function can still return False if for some reason access is not
granted).
'''
@functools.wraps(action)
def wrapper(context, data_dict):
return action(context, data_dict)
wrapper.auth_allow_anonymous_access = True
return wrapper
def auth_disallow_anonymous_access(action):
''' Flag an auth function as requiring a logged in user
This means that check_access will automatically raise a NotAuthorized
exception if an authenticated user is not provided in the context, without
calling the actual auth function.
'''
@functools.wraps(action)
def wrapper(context, data_dict):
return action(context, data_dict)
wrapper.auth_allow_anonymous_access = False
return wrapper
class UnknownValidator(Exception):
'''Exception raised when a requested validator function cannot be found.
'''
pass
_validators_cache = {}
def clear_validators_cache():
_validators_cache.clear()
# This function exists mainly so that validators can be made available to
# extensions via ckan.plugins.toolkit.
def get_validator(validator):
'''Return a validator function by name.
:param validator: the name of the validator function to return,
eg. ``'package_name_exists'``
:type validator: string
:raises: :py:exc:`~ckan.plugins.toolkit.UnknownValidator` if the named
validator is not found
:returns: the named validator function
:rtype: ``types.FunctionType``
'''
if not _validators_cache:
validators = _import_module_functions('ckan.lib.navl.validators')
_validators_cache.update(validators)
validators = _import_module_functions('ckan.logic.validators')
_validators_cache.update(validators)
_validators_cache.update({'OneOf': formencode.validators.OneOf})
converters = _import_module_functions('ckan.logic.converters')
_validators_cache.update(converters)
for plugin in p.PluginImplementations(p.IValidators):
for name, fn in plugin.get_validators().items():
if name in _validators_cache:
raise NameConflict(
'The validator %r is already defined' % (name,)
)
log.debug('Validator function {0} from plugin {1} was inserted'.format(name, plugin.name))
_validators_cache[name] = fn
try:
return _validators_cache[validator]
except KeyError:
raise UnknownValidator('Validator `%s` does not exist' % validator)
def model_name_to_class(model_module, model_name):
'''Return the class in model_module that has the same name as the received string.
Raises AttributeError if there's no model in model_module named model_name.
'''
try:
model_class_name = model_name.title()
return getattr(model_module, model_class_name)
except AttributeError:
raise ValidationError("%s isn't a valid model" % model_class_name)
def _import_module_functions(module_path):
'''Import a module and get the functions and return them in a dict'''
functions_dict = {}
module = __import__(module_path)
for part in module_path.split('.')[1:]:
module = getattr(module, part)
for k, v in module.__dict__.items():
try:
if v.__module__ != module_path:
continue
functions_dict[k] = v
except AttributeError:
pass
return functions_dict