forked from fedora-copr/copr
-
Notifications
You must be signed in to change notification settings - Fork 0
/
__init__.py
359 lines (299 loc) · 12.8 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
import json
import flask
import wtforms
import sqlalchemy
import inspect
from functools import wraps
from werkzeug.datastructures import ImmutableMultiDict, MultiDict
from werkzeug.exceptions import HTTPException, NotFound, GatewayTimeout
from sqlalchemy.orm.attributes import InstrumentedAttribute
from flask_restx import Api, Namespace, Resource
from coprs import app
from coprs.exceptions import (
AccessRestricted,
ActionInProgressException,
CoprHttpException,
InsufficientStorage,
ObjectNotFound,
BadRequest,
)
from coprs.logic.complex_logic import ComplexLogic
from coprs.helpers import streamed_json
apiv3_ns = flask.Blueprint("apiv3_ns", __name__, url_prefix="/api_3")
api = Api(
app=apiv3_ns,
version="1.0",
title="Copr APIv3",
description="See python client - <https://python-copr.readthedocs.io>",
doc="/docs",
)
# HTTP methods
GET = ["GET"]
POST = ["POST"]
PUT = ["POST", "PUT"]
DELETE = ["POST", "DELETE"]
def query_params():
def query_params_decorator(f):
@wraps(f)
def query_params_wrapper(*args, **kwargs):
sig = inspect.signature(f)
params = [x for x in sig.parameters]
params = list(set(params) - {"args", "kwargs"})
for arg in params:
if arg not in flask.request.args:
# If parameter is present in the URL path, we can use its
# value instead of failing that it is missing in query
# parameters, e.g. let's have a view decorated with these
# two routes:
# @foo_ns.route("/foo/bar/<int:build>/<chroot>")
# @foo_ns.route("/foo/bar") accepting ?build=X&chroot=Y
# @query_params()
# Then we need the following condition to get the first
# route working
if arg in flask.request.view_args:
continue
# If parameter has a default value, it is not required
if sig.parameters[arg].default == sig.parameters[arg].empty:
raise BadRequest("Missing argument {}".format(arg))
kwargs[arg] = flask.request.args.get(arg)
return f(*args, **kwargs)
return query_params_wrapper
return query_params_decorator
def pagination():
def pagination_decorator(f):
@wraps(f)
def pagination_wrapper(*args, **kwargs):
form = PaginationForm(flask.request.args)
if not form.validate():
raise CoprHttpException(form.errors)
kwargs.update(form.data)
return f(*args, **kwargs)
return pagination_wrapper
return pagination_decorator
def file_upload():
def file_upload_decorator(f):
@wraps(f)
def file_upload_wrapper(*args, **kwargs):
if "json" in flask.request.files:
data = json.loads(flask.request.files["json"].read()) or {}
tuples = [(k, v) for k, v in data.items()]
flask.request.form = ImmutableMultiDict(tuples)
return f(*args, **kwargs)
return file_upload_wrapper
return file_upload_decorator
class PaginationForm(wtforms.Form):
limit = wtforms.IntegerField("Limit", validators=[wtforms.validators.Optional()])
offset = wtforms.IntegerField("Offset", validators=[wtforms.validators.Optional()])
order = wtforms.StringField("Order by", validators=[wtforms.validators.Optional()])
order_type = wtforms.SelectField("Order type", validators=[wtforms.validators.Optional()],
choices=[("ASC", "ASC"), ("DESC", "DESC")], default="ASC")
def get_copr(ownername=None, projectname=None):
request = flask.request
ownername = ownername or request.form.get("ownername") or request.json["ownername"]
projectname = projectname or request.form.get("projectname") or request.json["projectname"]
return ComplexLogic.get_copr_by_owner_safe(ownername, projectname)
class Paginator(object):
LIMIT = None
OFFSET = 0
ORDER = "id"
def __init__(self, query, model, limit=None, offset=None, order=None, order_type=None, **kwargs):
self.query = query
self.model = model
self.limit = limit or self.LIMIT
self.offset = offset or self.OFFSET
self.order = order or self.ORDER
self.order_type = order_type
if not self.order_type:
# desc/asc unspecified, use some guessed defaults
if self.order == 'id':
self.order_type = 'DESC'
if self.order == 'name':
self.order_type = 'ASC'
def get(self):
return self.paginate_query(self.query)
def paginate_query(self, query):
"""
Return `self.query` with all pagination parameters (limit, offset,
order) but do not run it.
"""
order_attr = getattr(self.model, self.order, None)
if not order_attr:
msg = "Cannot order by {}, {} doesn't have such property".format(
self.order, self.model.__tablename__)
raise CoprHttpException(msg)
# This will happen when trying to sort by a property instead of
# a real database column
if not isinstance(order_attr, InstrumentedAttribute):
raise CoprHttpException("Cannot order by {}".format(self.order))
order_fun = (lambda x: x)
if self.order_type == 'ASC':
order_fun = sqlalchemy.asc
elif self.order_type == 'DESC':
order_fun = sqlalchemy.desc
return (query.order_by(order_fun(order_attr))
.limit(self.limit)
.offset(self.offset))
@property
def meta(self):
return {k: getattr(self, k) for k in ["limit", "offset", "order", "order_type"]}
def map(self, fun):
return [fun(x) for x in self.get()]
def to_dict(self):
return [x.to_dict() for x in self.get()]
class SubqueryPaginator(Paginator):
"""
Selecting rows with large offsets (400k+) is slower (~10 times) than
offset=0. There is not many options to get around it. To mitigate the
slowdown at least a little (~10%), we can filter, offset, and limit within
a subquery and then base the full-query on the subquery results.
"""
def __init__(self, query, subquery, *args, **kwargs):
super().__init__(query, *args, **kwargs)
self.pk = getattr(self.model, "id")
self.subquery = subquery.with_entities(self.pk)
def get(self):
subquery = self.paginate_query(self.subquery).subquery()
query = self.query.filter(self.pk.in_(subquery))
return query.all()
class ListPaginator(Paginator):
"""
The normal `Paginator` class works with a SQLAlchemy query object and
therefore can do limits and ordering on database level, which is ideal.
However, in some special cases, we already have a list of objects fetched
from database and need to adjust it based on user pagination preferences,
hence this special case of `Paginator` class.
It isn't efficient, it isn't pretty. Please use `Paginator` if you can.
"""
def get(self):
objects = self.query
reverse = self.order_type != "ASC"
if not hasattr(self.model, self.order):
msg = "Cannot order by {}, {} doesn't have such property".format(
self.order, self.model.__tablename__)
raise CoprHttpException(msg)
if self.order:
objects.sort(key=lambda x: getattr(x, self.order), reverse=reverse)
limit = None
if self.limit:
limit = self.offset + self.limit
return objects[self.offset : limit]
def editable_copr(f):
@wraps(f)
def wrapper(ownername, projectname, **kwargs):
copr = get_copr(ownername, projectname)
if not flask.g.user.can_edit(copr):
raise AccessRestricted(
"User '{0}' can not see permissions for project '{1}' "\
"(missing admin rights)".format(
flask.g.user.name,
'/'.join([ownername, projectname])
)
)
return f(copr, **kwargs)
return wrapper
def set_defaults(formdata, form_class):
"""
Take a `formdata` which can be `flask.request.form` or an output from
`get_form_compatible_data`, and `form_class` (not instance). This function
then goes through all fields defaults and update the `formdata` if those
fields weren't set from a user.
I don't understand why this is necessary, I would expect a `form.foo.data`
to return the request value or `default` but it doesn't seem to work like
that. See the `wtforms.fields.Field` documentation:
> default – The default value to assign to the field, if no form or object
> input is provided. May be a callable.
The **if no form or object input is provided** is really unexpected. We
always initialize our forms with request data, i.e. the defaults are never
used? IMHO a more reasonable approach would be doing this per field.
A field `default` value seems to be used only when rendering an empty form,
which is good enough for rendering an empty form into HTML (even though it
forces us to render all fields, which IMHO shouldn't be necessary). This
doesn't work at all for API where users send only a subset of form fields
and expect reasonable defaults for the rest.
"""
form = form_class(ImmutableMultiDict())
for field in form:
if field.default is None:
continue
if field.name in formdata.keys():
continue
formdata[field.name] = field.default
def streamed_json_array_response(array_or_generator, message, field="data"):
"""
Helper response to stream large JSON API arrays (ARRAY_OR_GENERATOR). We
keep the layout of the output like::
{
"output": "ok",
"message: MESSAGE,
"<FIELD>": [ ITEM, ITEM, ...,]
}
.. as it is expected by clients. We iterate continuously over the array
items (or fetch from generator), so we don't have to keep the large dataset
in memory (or wait till it is fully fetched from DB).
"""
start_string = (
'{{'
'"output": "ok",'
'"message": {message},'
'{field}: [\n'
).format(message=json.dumps(message),
field=json.dumps(field))
return streamed_json(array_or_generator, start_string, "]}")
def str_to_list(value, separator=None):
"""
We have a lot of module attributes that are stored as space-separeted
strings, their default value is `None` and we want to return them as lists.
It's tiresome to always check if the value is not `None` before splitting.
"""
if not value:
# Be careful:
# "".split() == []
# "".split(", ") == [""]
return []
return value.split(separator)
def reset_to_defaults(obj, form, rename_fields=None, more_fields=None):
"""
If the `form` contains a `reset` with a list of attributes that user wishes
to reset, set such values to their respective defaults. We find out the
default value from `obj`, which should be any database model.
Sometimes an attribute within the API schema can have a different name than
its model counterpart. For such cases this method takes an `rename_fields`,
function, see e.g. `apiv3_project_chroots.rename_fields`.
Use `more_fields` to specify a message saying where a user can find all the
possible attribute names
"""
reset = getattr(form, "reset_fields")
if not reset or not reset.data:
return
fields = str_to_list(reset.data)
# Some fields may have different name in the API schema than their
# respective attributes in the database models.
if rename_fields:
fields = rename_fields(dict.fromkeys(fields)).keys()
for field in fields:
try:
default = getattr(obj.__class__, field).default
value = default.arg if default else None
setattr(obj, field, value)
except AttributeError as ex:
msg = "Trying to reset an invalid attribute: {0}".format(field)
if more_fields:
msg += "\n{0}".format(more_fields)
raise BadRequest(msg) from ex
def rename_fields_helper(input_dict, replace):
"""
Transform the input dict or werkzeug.MultiDict into a MultiDict with renamed
keys according to the renames map.
"""
assert isinstance(input_dict, (dict, MultiDict))
output = MultiDict()
for key in input_dict.keys():
new_key = replace.get(key, key)
if isinstance(input_dict, MultiDict):
# Multiple values for one key.
values = input_dict.getlist(key)
else:
values = [input_dict[key]]
for value in values:
output.add(new_key, value)
return output