-
Notifications
You must be signed in to change notification settings - Fork 218
/
views.py
402 lines (318 loc) · 11.5 KB
/
views.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
import isodate
import datetime
import json
import math
import urllib
from collections import defaultdict
from django import http
from django.conf import settings
from django.contrib.auth.decorators import permission_required
from django.core.urlresolvers import reverse
from django.shortcuts import render
from django.views.decorators.http import require_POST
from django.utils import timezone
from waffle.decorators import waffle_switch
from ratelimit.decorators import ratelimit
from crashstats.api.views import has_permissions
from crashstats.crashstats import models, utils
from crashstats.crashstats.views import pass_default_context
from . import forms
from .form_fields import split_on_operator
from .models import SuperSearchFields, SuperSearchUnredacted, Query
DEFAULT_COLUMNS = (
'date',
'signature',
'product',
'version',
'build_id',
'platform',
)
DEFAULT_FACETS = (
'signature',
)
# Facetting on those fields doesn't provide useful information.
EXCLUDED_FIELDS_FROM_FACETS = (
'date',
'dump',
)
def get_allowed_fields(user):
return tuple(
x['name']
for x in SuperSearchFields().get().values()
if x['is_exposed']
and has_permissions(user, x['permissions_needed'])
)
def get_supersearch_form(request):
products = models.ProductsVersions().get()
versions = models.CurrentVersions().get()
platforms = models.Platforms().get()
all_fields = SuperSearchFields().get()
form = forms.SearchForm(
all_fields,
products,
versions,
platforms,
request.user,
request.GET
)
return form
def get_params(request):
form = get_supersearch_form(request)
if not form.is_valid():
return http.HttpResponseBadRequest(str(form.errors))
params = {}
for key in form.cleaned_data:
if hasattr(form.fields[key], 'prefixed_value'):
value = form.fields[key].prefixed_value
else:
value = form.cleaned_data[key]
params[key] = value
params['_facets'] = request.GET.getlist('_facets') or DEFAULT_FACETS
allowed_fields = get_allowed_fields(request.user)
# Make sure only allowed fields are used
params['_facets'] = [
x for x in params['_facets'] if x in allowed_fields
]
return params
@ratelimit(
key='ip',
rate=utils.ratelimit_rate,
method=ratelimit.ALL,
block=True
)
@pass_default_context
def search(request, default_context=None):
allowed_fields = get_allowed_fields(request.user)
context = default_context
context['possible_facets'] = [
{'id': x, 'text': x.replace('_', ' ')} for x in allowed_fields
if x not in EXCLUDED_FIELDS_FROM_FACETS
]
context['possible_columns'] = [
{'id': x, 'text': x.replace('_', ' ')} for x in allowed_fields
]
context['facets'] = request.GET.getlist('_facets') or DEFAULT_FACETS
context['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Fields data for the simple search UI.
form = get_supersearch_form(request)
context['simple_search_data'] = [
# field name, options, placeholder values
(x, form.fields[x].choices, [x[1] for x in form.fields[x].choices[:3]])
for x in settings.SIMPLE_SEARCH_FIELDS
]
return render(request, 'supersearch/search.html', context)
@ratelimit(
key='ip',
rate=utils.ratelimit_rate,
method=ratelimit.ALL,
block=True
)
def search_results(request):
'''Return the results of a search. '''
params = get_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, let's return it.
return params
data = {}
data['query'] = {
'total': 0,
'total_count': 0,
'total_pages': 0
}
allowed_fields = get_allowed_fields(request.user)
current_query = request.GET.copy()
if 'page' in current_query:
del current_query['page']
data['params'] = current_query.copy()
if '_columns' in data['params']:
del data['params']['_columns']
if '_facets' in data['params']:
del data['params']['_facets']
data['columns'] = request.GET.getlist('_columns') or DEFAULT_COLUMNS
# Make sure only allowed fields are used
data['columns'] = [
x for x in data['columns'] if x in allowed_fields
]
# Copy the list of columns so that they can differ.
params['_columns'] = list(data['columns'])
# The uuid is always displayed in the UI so we need to make sure it is
# always returned by the model.
if 'uuid' not in params['_columns']:
params['_columns'].append('uuid')
# The `uuid` field is a special case, it is always displayed in the first
# column of the table. Hence we do not want to show it again in the
# auto-generated list of columns, so we its name from the list of
# columns to display.
if 'uuid' in data['columns']:
data['columns'].remove('uuid')
try:
current_page = int(request.GET.get('page', 1))
except ValueError:
return http.HttpResponseBadRequest('Invalid page')
if current_page <= 0:
current_page = 1
results_per_page = 50
data['current_page'] = current_page
data['results_offset'] = results_per_page * (current_page - 1)
params['_results_number'] = results_per_page
params['_results_offset'] = data['results_offset']
data['current_url'] = '%s?%s' % (
reverse('supersearch.search'),
current_query.urlencode()
)
api = SuperSearchUnredacted()
try:
search_results = api.get(**params)
except models.BadStatusCodeError, e:
# We need to return the error message in some HTML form for jQuery to
# pick it up.
return http.HttpResponseBadRequest('<ul><li>%s</li></ul>' % e)
if 'signature' in search_results['facets']:
# Bugs for each signature
signatures = [h['term'] for h in search_results['facets']['signature']]
if signatures:
bugs = defaultdict(list)
bugs_api = models.Bugs()
for b in bugs_api.get(signatures=signatures)['hits']:
bugs[b['signature']].append(b['id'])
for hit in search_results['facets']['signature']:
sig = hit['term']
if sig in bugs:
if 'bugs' in hit:
hit['bugs'].extend(bugs[sig])
else:
hit['bugs'] = bugs[sig]
search_results['total_pages'] = int(math.ceil(
search_results['total'] / float(results_per_page)))
search_results['total_count'] = search_results['total']
data['query'] = search_results
data['report_list_query_string'] = urllib.urlencode(
utils.sanitize_dict(
get_report_list_parameters(params)
),
True
)
return render(request, 'supersearch/search_results.html', data)
@utils.json_view
def search_fields(request):
'''Return the JSON document describing the fields used by the JavaScript
dynamic_form library. '''
form = get_supersearch_form(request)
exclude = request.GET.getlist('exclude')
return form.get_fields_list(exclude=exclude)
def get_report_list_parameters(source):
'''Return a list of parameters that are compatible with the report/list
page. This is not ideal and cannot be fully compatible because we have
operators in supersearch and not in report/list.
'''
params = {}
for key, value in source.items():
if not value:
continue
if key in (
'hang_type',
'platform',
'process_type',
'product',
'reason',
):
params[key] = value
elif key == 'release_channel':
params['release_channels'] = value
elif key == 'build_id':
params['build_id'] = []
for build in value:
operator, build = split_on_operator(build)
if operator:
# The report/list/ page is unable to understand operators.
continue
params['build_id'].append(build)
if not params['build_id']:
del params['build_id']
elif key == 'version':
if 'product' in source:
params['version'] = []
for p in source['product']:
for v in value:
params['version'].append('%s:%s' % (p, v))
elif key == 'date':
lower = upper = up_ope = None
for dt in value:
operator, dt = split_on_operator(dt)
dt = isodate.parse_datetime(dt)
if lower is None or upper is None:
lower = upper = dt
up_ope = operator
elif lower > dt:
lower = dt
elif upper < dt:
upper = dt
up_ope = operator
def to_hours(delta):
return delta.days * 24 + delta.seconds / 3600
if lower == upper:
# there's only one date
if up_ope is not None and '<' in up_ope:
params['date'] = upper
else:
params['date'] = (
timezone.now()
)
params['range_value'] = to_hours(params['date'] - upper)
params['range_unit'] = 'hours'
else:
params['date'] = upper
params['range_value'] = to_hours(upper - lower)
params['range_unit'] = 'hours'
params['date'] = params['date'].strftime('%Y-%m-%d %H:%M:%S')
return params
@waffle_switch('!supersearch-custom-query-disabled')
@permission_required('crashstats.run_custom_queries')
@pass_default_context
def search_custom(request, default_context=None):
'''Return the basic search page, without any result. '''
error = None
query = None
params = get_params(request)
if isinstance(params, http.HttpResponseBadRequest):
# There was an error in the form, but we want to do the default
# behavior and just display an error message.
error = params
else:
# Get the JSON query that supersearch generates and show it.
params['_return_query'] = 'true'
api = SuperSearchUnredacted()
try:
query = api.get(**params)
except models.BadStatusCodeError, e:
error = e
schema = settings.ELASTICSEARCH_INDEX_SCHEMA
now = timezone.now()
possible_indices = []
for i in range(26):
index = (now - datetime.timedelta(weeks=i)).strftime(schema)
possible_indices.append({'id': index, 'text': index})
context = default_context
context['elasticsearch_indices'] = possible_indices
if query:
context['query'] = json.dumps(query['query'])
context['indices'] = ','.join(query['indices'])
context['error'] = error
return render(request, 'supersearch/search_custom.html', context)
@waffle_switch('!supersearch-custom-query-disabled')
@permission_required('crashstats.run_custom_queries')
@require_POST
@utils.json_view
def search_query(request):
form = forms.QueryForm(request.POST)
if not form.is_valid():
return http.HttpResponseBadRequest(form.errors)
api = Query()
try:
results = api.get(
query=form.cleaned_data['query'],
indices=form.cleaned_data['indices']
)
except models.BadStatusCodeError, e:
return http.HttpResponseBadRequest(e.message)
return results