96 changes: 67 additions & 29 deletions geonode/monitoring/collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,13 @@
from django.core.mail import EmailMultiAlternatives as EmailMessage
from django.utils.translation import ugettext_noop as _
from django.db.models import Max
from django.urls import resolve, Resolver404


from geonode.utils import raw_sql
from geonode.notifications_helper import send_notification
from geonode.monitoring import MonitoringAppConfig as AppConf
from geonode.monitoring.models import (Metric, MetricValue, RequestEvent,
from geonode.monitoring.models import (Metric, MetricValue, RequestEvent, MonitoredResource,
ExceptionEvent, EventType, NotificationCheck,)

from geonode.monitoring.utils import generate_periods, align_period_start, align_period_end
Expand All @@ -45,6 +46,7 @@
extract_event_types, extract_special_event_types,
get_resources_for_metric, get_labels_for_metric,
get_metric_names)
from geonode.base.models import ResourceBase
from geonode.utils import parse_datetime


Expand Down Expand Up @@ -394,41 +396,54 @@ def _key(v):
if metric.is_rate:
row = requests.aggregate(value=models.Avg(column_name))
row['samples'] = requests.count()
row['label'] = 'rate'
row['label'] = Metric.TYPE_RATE
q = [row]

elif metric.is_count:
q = []
values = requests.distinct(
column_name).values_list(column_name, flat=True)
for v in values:
row = requests.filter(**{column_name: v})\
.aggregate(value=models.Sum(column_name),
samples=models.Count(column_name))
rqs = requests.filter(**{column_name: v})
row = rqs.aggregate(
value=models.Sum(column_name),
samples=models.Count(column_name)
)
row['label'] = v
q.append(row)

q.sort(key=_key)
q.reverse()

elif metric.is_value:

q = []
values = requests.distinct(
column_name).values_list(column_name, flat=True)
is_user_metric = column_name == "user_identifier"
if is_user_metric:
values = requests.distinct(
column_name).values_list(column_name, "user_username")
else:
values = requests.distinct(
column_name).values_list(column_name, flat=True)
for v in values:
row = requests.filter(**{column_name: v})\
.aggregate(value=models.Count(column_name),
samples=models.Count(column_name))
value = v
if is_user_metric:
value = v[0]
rqs = requests.filter(**{column_name: value})
row = rqs.aggregate(
value=models.Count(column_name),
samples=models.Count(column_name)
)
row['label'] = v
q.append(row)
q.sort(key=_key)
q.reverse()

elif metric.is_value_numeric:
q = []
row = requests.aggregate(value=models.Max(column_name),
samples=models.Count(column_name))
row['label'] = v # TODO: v could be undefined
row['label'] = Metric.TYPE_VALUE_NUMERIC
q.append(row)

else:
raise ValueError("Unsupported metric type: {}".format(metric.type))
rows = q[:100]
Expand Down Expand Up @@ -677,14 +692,14 @@ def get_metrics_data(self, metric_name,
col = 'mv.value_num'
agg_f = self.get_aggregate_function(col, metric_name, service)
has_agg = agg_f != col
group_by_map = {'resource': {'select': ['mr.id', 'mr.type', 'mr.name', ],
group_by_map = {'resource': {'select': ['mr.id', 'mr.type', 'mr.name', 'mr.resource_id'],
'from': ['join monitoring_monitoredresource mr on (mv.resource_id = mr.id)'],
'where': ['and mv.resource_id is not NULL'],
'order_by': None,
'grouper': ['resource', 'name', 'type', 'id', ],
'grouper': ['resource', 'name', 'type', 'id', 'resource_id'],
},
# group by resource, but do not show labels. number of unique labels will be used as val
'resource_no_label': {'select_only': ['mr.id', 'mr.type', 'mr.name',
'resource_no_label': {'select_only': ['mr.id', 'mr.type', 'mr.name', 'mr.resource_id',
'count(distinct(ml.name)) as val',
'count(1) as metric_count',
'sum(samples_count) as samples_count',
Expand All @@ -696,7 +711,7 @@ def get_metrics_data(self, metric_name,
'where': ['and mv.resource_id is not NULL'],
'order_by': ['val desc'],
'group_by': ['mr.id', 'mr.type', 'mr.name'],
'grouper': ['resource', 'name', 'type', 'id', ],
'grouper': ['resource', 'name', 'type', 'id', 'resource_id'],
},
'event_type': {'select_only': ['ev.name as event_type', 'count(1) as val',
'count(1) as metric_count',
Expand Down Expand Up @@ -772,7 +787,7 @@ def get_metrics_data(self, metric_name,
if group_by not in ('event_type', 'event_type_on_label',) and event_type is None:
event_type = EventType.get(EventType.EVENT_ALL)

if event_type:
if event_type and metric_name not in ['uptime', ]:
q_where.append(' and mv.event_type_id = %(event_type)s ')
params['event_type'] = event_type.id

Expand All @@ -781,20 +796,15 @@ def get_metrics_data(self, metric_name,
params['label'] = label.id
# if not group_by and not resource:
# resource = MonitoredResource.get('', '', or_create=True)
if resource and not group_by:
q_from.append('join monitoring_monitoredresource mr on '
'(mv.resource_id = mr.id and mr.id = %(resource_id)s) ')
params['resource_id'] = resource.id

if label and has_agg:
q_group.extend(['ml.name'])
if resource and group_by in ('resource', 'resource_no_label',):
raise ValueError(
"Cannot use resource and group by resource at the same time")
if resource and has_agg:
q_group.append('mr.name')
# group returned columns into a dict
# config in grouping map: target_column = {source_column1: val, ...}

if label and has_agg:
q_group.extend(['ml.name'])

grouper = None
if group_by:
group_by_cfg = group_by_map[group_by]
Expand All @@ -814,12 +824,25 @@ def get_metrics_data(self, metric_name,
q_group.extend(group_by_cfg['select'])
grouper = group_by_cfg['grouper']

if resource_type:
if resource_type and not resource:
if not [mr for mr in q_from if 'monitoring_monitoredresource' in mr]:
q_from.append('join monitoring_monitoredresource mr on mv.resource_id = mr.id ')
q_where.append(' and mr.type = %(resource_type)s ')
params['resource_type'] = resource_type

if resource and group_by in ('resource', 'resource_no_label',):
raise ValueError(
"Cannot use resource and group by resource at the same time")
elif resource:
if not [mr for mr in q_from if 'monitoring_monitoredresource' in mr]:
q_from.append('join monitoring_monitoredresource mr on mv.resource_id = mr.id ')
q_where.append(' and mr.id = %(resource_id)s ')
params['resource_id'] = resource.id

if 'ml.name' in q_group:
q_select.append(', max(ml.user) as user')
# q_group.extend(['ml.user']) not needed

if q_group:
q_group = [' group by ', ','.join(q_group)]
if q_order_by:
Expand All @@ -832,7 +855,22 @@ def postproc(row):
t = {}
tcol = grouper[0]
for scol in grouper[1:]:
t[scol] = row.pop(scol)
if scol == 'resource_id':
if scol in row:
r_id = row.pop(scol)
try:
rb = ResourceBase.objects.get(id=r_id)
t['href'] = rb.detail_url
except BaseException:
t['href'] = ""
else:
t[scol] = row.pop(scol)
if scol == 'type' and t[scol] == MonitoredResource.TYPE_URL:
try:
resolve(t['name'])
t['href'] = t['name']
except Resolver404:
t['href'] = ""
row[tcol] = t
return row

Expand Down
46,539 changes: 46,539 additions & 0 deletions geonode/monitoring/fixtures/metric_data.json

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions geonode/monitoring/middleware.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,12 @@ def should_process(request):
return True

@staticmethod
def register_event(request, event_type, resource_type, resource_name):
def register_event(request, event_type, resource_type, resource_name, resource_id):
m = getattr(request, '_monitoring', None)
if not m:
return
events = m['events']
events.append((event_type, resource_type, resource_name,))
events.append((event_type, resource_type, resource_name, resource_id,))

def register_request(self, request, response):
if self.service:
Expand Down Expand Up @@ -125,13 +125,13 @@ def process_request(self, request):
if settings.USER_ANALYTICS_ENABLED:
meta.update({
'user_identifier': hashlib.sha256(request.session.session_key or '').hexdigest(),
'user_username': request.user.username if request.user.is_authenticated() else None
'user_username': request.user.username if request.user.is_authenticated() else 'AnonymousUser'
})

request._monitoring = meta

def register_event(event_type, resource_type, name):
self.register_event(request, event_type, resource_type, name)
def register_event(event_type, resource_type, name, resource_id):
self.register_event(request, event_type, resource_type, name, resource_id)

request.register_event = register_event

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2019-08-26 09:20
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('monitoring', '0026_auto_20190821_0736'),
]

operations = [
migrations.AddField(
model_name='monitoredresource',
name='resource_id',
field=models.IntegerField(blank=True, null=True),
),
]
25 changes: 25 additions & 0 deletions geonode/monitoring/migrations/0028_auto_20190830_1018.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2019-08-30 10:18
from __future__ import unicode_literals

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('monitoring', '0027_monitoredresource_resource_id'),
]

operations = [
migrations.AddField(
model_name='metriclabel',
name='user',
field=models.CharField(blank=True, default=None, max_length=150, null=True),
),
migrations.AlterField(
model_name='eventtype',
name='name',
field=models.CharField(choices=[(b'OWS:TMS', b'TMS'), (b'OWS:WMS-C', b'WMS-C'), (b'OWS:WMTS', b'WMTS'), (b'OWS:WCS', b'WCS'), (b'OWS:WFS', b'WFS'), (b'OWS:WMS', b'WMS'), (b'OWS:WPS', b'WPS'), (b'other', b'Not OWS'), (b'OWS:ALL', b'Any OWS'), (b'all', b'All'), (b'create', b'Create'), (b'upload', b'Upload'), (b'change', b'Change'), (b'change_metadata', b'Change Metadata'), (b'view_metadata', b'View Metadata'), (b'view', b'View'), (b'download', b'Download'), (b'publish', b'Publish'), (b'remove', b'Remove'), (b'geoserver', b'Geoserver event')], max_length=16, unique=True),
),
]
34 changes: 30 additions & 4 deletions geonode/monitoring/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ class MonitoredResource(models.Model):
blank=False,
choices=TYPES,
default=TYPE_EMPTY)
resource_id = models.IntegerField(null=True, blank=True)

class Meta:
unique_together = (('name', 'type',),)
Expand Down Expand Up @@ -350,7 +351,7 @@ class EventType(models.Model):
EVENT_ALL = 'all' # all events - baseline: ows + non-ows

EVENT_TYPES = zip(['OWS:{}'.format(ows) for ows in _ows_types], _ows_types) + \
[(EVENT_OTHER, _("Non-OWS"))] +\
[(EVENT_OTHER, _("Not OWS"))] +\
[(EVENT_OWS, _("Any OWS"))] +\
[(EVENT_ALL, _("All"))] +\
[(EVENT_CREATE, _("Create"))] +\
Expand Down Expand Up @@ -501,6 +502,18 @@ def _get_resources(cls, type_name, resources_list):
out.append(rinst)
return out

@classmethod
def _get_or_create_resources(cls, res_name, res_type, res_id):
out = []
r, _ = MonitoredResource.objects.get_or_create(
name=res_name, type=res_type
)
if r and res_id:
r.resource_id = res_id
r.save()
out.append(r)
return out

@classmethod
def _get_geonode_resources(cls, request):
"""
Expand All @@ -513,8 +526,8 @@ def _get_geonode_resources(cls, request):
# res = rqmeta['resources'].get(type_name) or []
# resources.extend(cls._get_resources(type_name, res))

for evt_type, res_type, res_name in events:
resources.extend(cls._get_resources(res_type, [res_name]))
for evt_type, res_type, res_name, res_id in events:
resources.extend(cls._get_or_create_resources(res_name, res_type, res_id))

return resources

Expand Down Expand Up @@ -874,6 +887,11 @@ def expose(self):
class MetricLabel(models.Model):

name = models.TextField(null=False, blank=True, default='')
user = models.CharField(
max_length=150,
default=None,
null=True,
blank=True)

def __str__(self):
return 'Metric Label: {}'.format(self.name)
Expand Down Expand Up @@ -948,7 +966,15 @@ def add(cls, metric, valid_from, valid_to, service, label,
service_metric = ServiceTypeMetric.objects.get(
service_type=service.service_type, metric__name=metric)

label, _ = MetricLabel.objects.get_or_create(name=label or 'count')
label_name = label
label_user = None
if label and isinstance(label, tuple):
label_name = label[0]
label_user = label[1]
label, c = MetricLabel.objects.get_or_create(name=label_name or 'count')
if c and label_user:
label.user = label_user
label.save()
if event_type:
if not isinstance(event_type, EventType):
event_type = EventType.get(event_type)
Expand Down
2 changes: 1 addition & 1 deletion geonode/monitoring/static/monitoring/bundle.js

Large diffs are not rendered by default.

1,294 changes: 1,091 additions & 203 deletions geonode/monitoring/tests/integration.py

Large diffs are not rendered by default.

19 changes: 18 additions & 1 deletion geonode/monitoring/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from django.db.models.fields.related import RelatedField

from geonode.monitoring.models import RequestEvent, ExceptionEvent
from geonode.settings import DATETIME_INPUT_FORMATS


GS_FORMAT = '%Y-%m-%dT%H:%M:%S' # 2010-06-20T2:00:00
Expand Down Expand Up @@ -139,7 +140,7 @@ def get_requests(self, format=None, since=None, until=None):
if data:
yield data
else:
log.href("Skipping payload for {}".format(href))
log.warning("Skipping payload for {}".format(href))

def get_request(self, href, format=format):
username = settings.OGC_SERVER['default']['USER']
Expand Down Expand Up @@ -353,3 +354,19 @@ def dump(obj, additional_fields=tuple()):
'seconds': val.total_seconds()}
out[fname] = val
return out


def extend_datetime_input_formats(formats):
"""
Add new DateTime input formats
:param formats: input formats yoy want to add (tuple or list)
:return: extended input formats
"""
input_formats = DATETIME_INPUT_FORMATS
if isinstance(input_formats, tuple):
input_formats += tuple(formats)
elif isinstance(input_formats, list):
input_formats.extend(formats)
else:
raise ValueError("Input parameter must be tuple or list.")
return input_formats
47 changes: 41 additions & 6 deletions geonode/monitoring/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from django.core.urlresolvers import reverse
from django.core.management import call_command
from django.views.decorators.csrf import csrf_exempt
from geonode.decorators import view_decorator, superuser_protected

from geonode.utils import json_response
from geonode.monitoring.collector import CollectorAPI
Expand All @@ -46,7 +47,7 @@
MetricNotificationCheck,
)
from geonode.monitoring.models import do_autoconfigure
from geonode.monitoring.utils import TypeChecks, dump
from geonode.monitoring.utils import TypeChecks, dump, extend_datetime_input_formats
from geonode.monitoring.service_handlers import exposes

# Create your views here.
Expand Down Expand Up @@ -100,8 +101,14 @@ def get(self, *args, **kwargs):


class _ValidFromToLastForm(forms.Form):
valid_from = forms.DateTimeField(required=False)
valid_to = forms.DateTimeField(required=False)
valid_from = forms.DateTimeField(
required=False,
input_formats=extend_datetime_input_formats(['%Y-%m-%dT%H:%M:%S.%fZ'])
)
valid_to = forms.DateTimeField(
required=False,
input_formats=extend_datetime_input_formats(['%Y-%m-%dT%H:%M:%S.%fZ'])
)
interval = forms.IntegerField(min_value=60, required=False)
last = forms.IntegerField(min_value=60, required=False)

Expand Down Expand Up @@ -251,6 +258,7 @@ def get(self, request, *args, **kwargs):
return json_response(data)


@view_decorator(superuser_protected, subclass=True)
class ResourcesList(FilteredView):

filter_form = ResourcesFilterForm
Expand Down Expand Up @@ -289,6 +297,7 @@ def get_queryset(self, metric_name=None,
return q


@view_decorator(superuser_protected, subclass=True)
class ResourceTypesList(FilteredView):

output_name = 'resource_types'
Expand All @@ -301,7 +310,7 @@ def get(self, request, *args, **kwargs):
'status': 'errors',
'errors': f.errors},
status=400)
out = [{"name": mrt[0], "type": mrt[1]} for mrt in MonitoredResource.TYPES]
out = [{"name": mrt[0], "type_label": mrt[1]} for mrt in MonitoredResource.TYPES]
data = {self.output_name: out,
'success': True,
'errors': {},
Expand All @@ -311,6 +320,7 @@ def get(self, request, *args, **kwargs):
return json_response(data)


@view_decorator(superuser_protected, subclass=True)
class LabelsList(FilteredView):

filter_form = LabelsFilterForm
Expand Down Expand Up @@ -341,15 +351,40 @@ def get_queryset(self, metric_name, valid_from,
return q


@view_decorator(superuser_protected, subclass=True)
class EventTypeList(FilteredView):

fields_map = (('name', 'name',),)
fields_map = (('name', 'name',), ('type_label', 'type_label',),)
output_name = 'event_types'

def get_queryset(self, **kwargs):
return EventType.objects.all()

def get(self, request, *args, **kwargs):
qargs = self.get_filter_args(request)
if self.errors:
return json_response({'success': False,
'status': 'errors',
'errors': self.errors},
status=400)
q = self.get_queryset(**qargs)
from_fields = [f[0] for f in self.fields_map]
to_fields = [f[1] for f in self.fields_map]
labels = dict(EventType.EVENT_TYPES)
out = [dict(zip(
to_fields,
(getattr(item, f) if f != 'type_label' else labels[getattr(item, 'name')] for f in from_fields)
)) for item in q]
data = {self.output_name: out,
'success': True,
'errors': {},
'status': 'ok'}
if self.output_name != 'data':
data['data'] = {'key': self.output_name}
return json_response(data)


@view_decorator(superuser_protected, subclass=True)
class MetricDataView(View):

def get_filters(self, **kwargs):
Expand All @@ -362,7 +397,7 @@ def get_filters(self, **kwargs):
out.update(f.cleaned_data)
return out

def get(self, *args, **kwargs):
def get(self, request, *args, **kwargs):
filters = self.get_filters(**kwargs)
if self.errors:
return json_response({'status': 'error',
Expand Down
4 changes: 2 additions & 2 deletions geonode/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -461,8 +461,8 @@

# Django 1.11 ParallelTestSuite
TEST_RUNNER = 'geonode.tests.suite.runner.GeoNodeBaseSuiteDiscoverRunner'
TEST_RUNNER_KEEPDB = 0
TEST_RUNNER_PARALLEL = 1
TEST_RUNNER_KEEPDB = os.environ.get('TEST_RUNNER_KEEPDB', 0)
TEST_RUNNER_PARALLEL = os.environ.get('TEST_RUNNER_PARALLEL', 1)

# GeoNode test suite
# TEST_RUNNER = 'geonode.tests.suite.runner.DjangoParallelTestSuiteRunner'
Expand Down
7 changes: 6 additions & 1 deletion geonode/upload/tests/test_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,15 @@
'guardian.backends.ObjectPermissionBackend'
)

SESSION_EXPIRED_CONTROL_ENABLED = False
SESSION_ENGINE = 'django.contrib.sessions.backends.db'
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer'

SESSION_EXPIRED_CONTROL_ENABLED = False
if 'geonode.security.middleware.SessionControlMiddleware' in MIDDLEWARE_CLASSES:
_middleware = list(MIDDLEWARE_CLASSES)
_middleware.remove('geonode.security.middleware.SessionControlMiddleware')
MIDDLEWARE_CLASSES = tuple(_middleware)

# Backend
DATABASES = {
'default': {
Expand Down
4 changes: 2 additions & 2 deletions geonode/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,10 +311,10 @@ def bbox_to_projection(native_bbox, target_srid=4326):
projected_bbox = [str(x) for x in poly.extent]
# Must be in the form : [x0, x1, y0, y1, EPSG:<target_srid>)
return tuple([projected_bbox[0], projected_bbox[2], projected_bbox[1], projected_bbox[3]]) + \
("EPSG:%s" % poly.srid,)
("EPSG:%s" % target_srid,)
except BaseException:
tb = traceback.format_exc()
logger.debug(tb)
logger.info(tb)

return native_bbox

Expand Down
2 changes: 1 addition & 1 deletion pavement.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@

try:
from geonode.settings import TEST_RUNNER_KEEPDB, TEST_RUNNER_PARALLEL
_keepdb = '-k' if TEST_RUNNER_KEEPDB else ''
_keepdb = '--keepdb' if TEST_RUNNER_KEEPDB else ''
_parallel = ('--parallel=%s' % TEST_RUNNER_PARALLEL) if TEST_RUNNER_PARALLEL else ''
except BaseException:
_keepdb = ''
Expand Down