Skip to content

Commit

Permalink
[Fixes #4896] fixing data in response
Browse files Browse the repository at this point in the history
  • Loading branch information
gioscarda committed Sep 13, 2019
1 parent cfb4de1 commit 5156cbd
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 49 deletions.
25 changes: 15 additions & 10 deletions geonode/monitoring/collector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,6 @@ def process_host_geonode(self, service, data, valid_from, valid_to):
.strftime("%Y-%m-%d %H:%M:%S")).replace(tzinfo=utc)
valid_from = align_period_start(collected_at, service.check_interval)
valid_to = align_period_end(collected_at, service.check_interval)

mdefaults = {'valid_from': valid_from,
'valid_to': valid_to,
'resource': None,
Expand Down Expand Up @@ -483,14 +482,13 @@ def process_requests(self, service, requests, valid_from, valid_to):

def set_error_values(self, requests, valid_from, valid_to,
service=None, resource=None, event_type=None):
with_errors = requests.filter(exceptions__isnull=False)
with_errors = requests.exclude(exceptions=None)
if not with_errors.exists():
return

labels = ExceptionEvent.objects.filter(request__in=with_errors)\
.distinct()\
.values_list('error_type', flat=True)

defaults = {'valid_from': valid_from,
'valid_to': valid_to,
'resource': resource,
Expand Down Expand Up @@ -637,8 +635,8 @@ def get_metrics_for(self, metric_name,
interval = timedelta(seconds=interval)
metric = Metric.objects.get(name=metric_name)
out = {'metric': metric.name,
'input_valid_from': valid_from,
'input_valid_to': valid_to,
'input_valid_from': valid_from.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'input_valid_to': valid_to.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'interval': interval.total_seconds(),
'label': label.name if label else None,
'type': metric.type,
Expand All @@ -656,8 +654,11 @@ def get_metrics_for(self, metric_name,
resource=resource,
resource_type=resource_type,
group_by=group_by)
out['data'].append(
{'valid_from': pstart, 'valid_to': pend, 'data': pdata})
out['data'].append({
'valid_from': pstart.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'valid_to': pend.strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
'data': pdata
})
return out

def get_aggregate_function(self, column_name, metric_name, service=None):
Expand Down Expand Up @@ -687,6 +688,7 @@ def get_metrics_data(self, metric_name,
"""
Returns metric values for metric within given time span
"""
utc = pytz.utc
params = {}
col = 'mv.value_num'
agg_f = self.get_aggregate_function(col, metric_name, service)
Expand Down Expand Up @@ -831,9 +833,10 @@ def get_metrics_data(self, metric_name,
if metric_name == 'uptime':
q_where = ['where', 'm.name = %(metric_name)s']
q_group = ['ml.name']

params.update({'metric_name': metric_name,
'valid_from': valid_from.strftime('%Y-%m-%d %H:%M:%S'),
'valid_to': valid_to.strftime('%Y-%m-%d %H:%M:%S')})
'valid_from': valid_from.replace(tzinfo=utc).isoformat(),
'valid_to': valid_to.replace(tzinfo=utc).isoformat()})

q_order_by = ['val desc']

Expand All @@ -857,7 +860,9 @@ def get_metrics_data(self, metric_name,
'event_type_on_user'):
event_type = EventType.get(EventType.EVENT_ALL)

if event_type and metric_name not in BuiltIns.host_metrics:
exclude_ev_type = BuiltIns.host_metrics + ('response.error.count',)

if event_type and metric_name not in exclude_ev_type:
q_where.append(' and mv.event_type_id = %(event_type)s ')
params['event_type'] = event_type.id

Expand Down
7 changes: 6 additions & 1 deletion geonode/monitoring/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,7 @@ def from_geoserver(cls, service, request_data, received=None):
event_type = EventType.get('OWS:{}'.format(event_type_name.upper()))
else:
event_type = EventType.get(EventType.EVENT_GEOSERVER)

data = {'created': start_time,
'received': received,
'host': rd['host'],
Expand Down Expand Up @@ -971,7 +972,11 @@ def add(cls, metric, valid_from, valid_to, service, label,
if label and isinstance(label, tuple):
label_name = label[0]
label_user = label[1]
label, c = MetricLabel.objects.get_or_create(name=label_name or 'count')
try:
label, c = MetricLabel.objects.get_or_create(name=label_name or 'count')
except MetricLabel.MultipleObjectsReturned:
c = False
label = MetricLabel.objects.filter(name=label_name).first()
if c and label_user:
label.user = label_user
label.save()
Expand Down
76 changes: 38 additions & 38 deletions geonode/monitoring/tests/integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -986,8 +986,8 @@ def test_layer_view_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# check data
Expand Down Expand Up @@ -1047,8 +1047,8 @@ def test_layer_upload_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# check data
Expand Down Expand Up @@ -1108,8 +1108,8 @@ def test_layer_view_metadata_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# check data
Expand Down Expand Up @@ -1161,8 +1161,8 @@ def test_layer_change_metadata_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# check data
Expand Down Expand Up @@ -1214,8 +1214,8 @@ def test_layer_download_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# check data
Expand Down Expand Up @@ -1275,8 +1275,8 @@ def test_map_create_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# check data
Expand Down Expand Up @@ -1318,8 +1318,8 @@ def test_map_change_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"]
Expand Down Expand Up @@ -1352,8 +1352,8 @@ def test_document_upload_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"]
Expand Down Expand Up @@ -1386,8 +1386,8 @@ def test_document_view_metadata_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"]
Expand Down Expand Up @@ -1422,8 +1422,8 @@ def test_document_change_metadata_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"]
Expand Down Expand Up @@ -1455,8 +1455,8 @@ def test_document_download_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# check data
Expand Down Expand Up @@ -1603,8 +1603,8 @@ def test_url_view_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# check data
Expand Down Expand Up @@ -1796,8 +1796,8 @@ def test_users_count_for_resource_endpoint(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 31536000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"][0]["data"]
Expand Down Expand Up @@ -1839,8 +1839,8 @@ def test_resources_count_endpoint(self):
self.assertEqual(out["data"]["metric"], 'request.count')
self.assertEqual(out["data"]["interval"], 31536000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'count')
data = out["data"]["data"][0]["data"]
Expand Down Expand Up @@ -2051,8 +2051,8 @@ def test_event_type_on_label_endpoint(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 31536000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"][0]["data"]
Expand Down Expand Up @@ -2142,8 +2142,8 @@ def test_event_type_on_user_endpoint(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 31536000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"][0]["data"]
Expand Down Expand Up @@ -2185,8 +2185,8 @@ def test_unique_visitors_count_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"]
Expand Down Expand Up @@ -2246,8 +2246,8 @@ def test_anonymous_sessions_count_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
data = out["data"]["data"]
Expand Down Expand Up @@ -2343,8 +2343,8 @@ def test_unique_visitors_list_endpoints(self):
self.assertEqual(out["data"]["metric"], 'request.users')
self.assertEqual(out["data"]["interval"], 2628000)
self.assertEqual(out["data"]["label"], None)
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00Z')
self.assertEqual(out["data"]["input_valid_from"], '2018-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["input_valid_to"], '2019-09-11T20:00:00.000000Z')
self.assertEqual(out["data"]["axis_label"], 'Count')
self.assertEqual(out["data"]["type"], 'value')
# # check data
Expand Down

0 comments on commit 5156cbd

Please sign in to comment.