312 changes: 263 additions & 49 deletions docs/intermediate/monitoring/user_analytics/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ Requests
* get number of unique tracking ids for urls

``GET /monitoring/api/metric_data/request.users/?``
``last=(x*86400)&interval=86400&group_by=resource_no_label&resource_type=url&event_type=other``
``last=(x*86400)&interval=86400&group_by=resource_on_label&resource_type=url&event_type=other``

.. code-block:: json
Expand Down Expand Up @@ -135,71 +135,125 @@ Requests
``GET /monitoring/api/metric_data/request.users/?``
``last=86400&interval=86400&group_by=event_type_on_label``

* to get number of unique tracking ids for specific resource type
* to get number of unique tracking ids for each event_type on a given resource type

``GET /monitoring/api/metric_data/request.users/?``
``last=86400&interval=86400&group_by=event_type_on_label&resource_type=url``

.. code-block:: json
{
"data": {
"input_valid_from": "2018-07-11T17:54:41.467Z",
"input_valid_to": "2018-07-12T17:54:41.467Z",
"data": [
{
"valid_from": "2018-07-11T17:54:41.467Z",
"data": [
{
"samples_count": 5,
"event_type": "all",
"val": 2,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
},
{
"samples_count": 5,
"event_type": "other",
"val": 2,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
},
{
"samples_count": 5,
"event_type": "view",
"val": 2,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
}
],
"valid_to": "2018-07-12T17:54:41.467Z"
}
],
"metric": "request.users",
"interval": 86400.0,
"type": "value",
"axis_label": "Count",
"label": null
}
{
"data": {
"input_valid_from": "2018-07-11T17:54:41.467Z",
"input_valid_to": "2018-07-12T17:54:41.467Z",
"data": [
{
"valid_from": "2018-07-11T17:54:41.467Z",
"data": [
{
"samples_count": 5,
"event_type": "all",
"val": 2,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
},
{
"samples_count": 5,
"event_type": "other",
"val": 2,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
},
{
"samples_count": 5,
"event_type": "view",
"val": 2,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
}
],
"valid_to": "2018-07-12T17:54:41.467Z"
}
],
"metric": "request.users",
"interval": 86400.0,
"type": "value",
"axis_label": "Count",
"label": null
}
}
* to get number of unique users for each event type on specific resource type

``GET /monitoring/api/metric_data/request.users/?``
``last=86400&interval=86400&group_by=event_type_on_user&resource_type=url``

.. code-block:: json
{
"data": {
"input_valid_from": "2018-07-11T17:54:41.467Z",
"input_valid_to": "2018-07-12T17:54:41.467Z",
"data": [
{
"valid_from": "2018-07-11T17:54:41.467Z",
"data": [
{
"samples_count": 5,
"event_type": "all",
"val": 2,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
},
{
"samples_count": 5,
"event_type": "other",
"val": 1,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
},
{
"samples_count": 5,
"event_type": "view",
"val": 1,
"min": "1.0000",
"max": "1.0000",
"sum": "5.0000",
"metric_count": 5
}
],
"valid_to": "2018-07-12T17:54:41.467Z"
}
],
"metric": "request.users",
"interval": 86400.0,
"type": "value",
"axis_label": "Count",
"label": null
}
}
4. total number of unique sessions per event_type and single resource: let me see what was the most visited map page in this day, or what was the most downloaded document, what was the most requested ows layer, etc.

* list of most visited resources of `url` type

``GET /monitoring/api/metric_data/request.users/?``
``last=86400&interval=86400&group_by=resource_no_label&resource_type=url``
``last=86400&interval=86400&group_by=resource_on_label&resource_type=url``

* list of unique tracking ids for each resource (can be narrowed down to specific resource type with `resource_type` values).

``GET /monitoring/api/metric_data/request.users/?``
``last=86400&interval=86400&group_by=resource_no_label``
``last=86400&interval=86400&group_by=resource_on_label``

.. code-block:: json
Expand Down Expand Up @@ -500,3 +554,163 @@ Requests
"label": null
}
}
8. total number of unique users for each monitored resource.

``GET /monitoring/api/metric_data/request.users/``
``?last=31536000&interval=31536000&group_by=resource_on_user``

.. code-block:: json
{
"data": {
"input_valid_from": "2018-09-10T14:15:39.454Z",
"input_valid_to": "2019-09-10T14:15:39.454Z",
"data": [
{
"valid_from": "2018-09-10T14:15:39.454Z",
"data": [
{
"resource": {
"href": "/",
"type": "url",
"name": "/",
"id": 1
},
"metric_count": 36,
"val": 4,
"max": "2.0000",
"sum": "35.0000",
"min": "0.0000",
"samples_count": 35
},
{
"resource": {
"href": "/maps/",
"type": "url",
"name": "/maps/",
"id": 3
},
"metric_count": 3,
"val": 2,
"max": "1.0000",
"sum": "3.0000",
"min": "1.0000",
"samples_count": 3
},
{
"resource": {
"href": "",
"type": "layer",
"name": "geonode:railways",
"id": 4
},
"metric_count": 5,
"val": 2,
"max": "2.0000",
"sum": "3.0000",
"min": "0.0000",
"samples_count": 3
},
{
"resource": {
"href": "/layers/",
"type": "url",
"name": "/layers/",
"id": 2
},
"metric_count": 4,
"val": 1,
"max": "1.0000",
"sum": "4.0000",
"min": "1.0000",
"samples_count": 4
},
{
"resource": {
"href": "/documents/2",
"type": "document",
"name": "test_doc_1.txt",
"id": 5
},
"metric_count": 2,
"val": 1,
"max": "2.0000",
"sum": "4.0000",
"min": "2.0000",
"samples_count": 4
},
{
"resource": {
"href": "/maps/3",
"type": "map",
"name": "test_map",
"id": 6
},
"metric_count": 1,
"val": 1,
"max": "1.0000",
"sum": "1.0000",
"min": "1.0000",
"samples_count": 1
},
{
"resource": {
"href": "",
"type": "layer",
"name": "geonode:waterways",
"id": 7
},
"metric_count": 2,
"val": 1,
"max": "2.0000",
"sum": "2.0000",
"min": "0.0000",
"samples_count": 2
}
],
"valid_to": "2019-09-10T14:15:39.454Z"
}
],
"metric": "request.users",
"interval": 31536000,
"type": "value",
"axis_label": "Count",
"label": null
}
}
9. total number of resource monitored in a given time range.

``GET /monitoring/api/metric_data/request.users/``
``?last=31536000&interval=31536000&group_by=count_on_resource``

.. code-block:: json
{
"data": {
"input_valid_from": "2018-09-10T14:20:27.335Z",
"input_valid_to": "2019-09-10T14:20:27.335Z",
"data": [
{
"valid_from": "2018-09-10T14:20:27.335Z",
"data": [
{
"samples_count": 52,
"val": 7,
"min": "0.0000",
"max": "2.0000",
"sum": "52.0000",
"metric_count": 53
}
],
"valid_to": "2019-09-10T14:20:27.335Z"
}
],
"metric": "request.users",
"interval": 31536000,
"type": "value",
"axis_label": "Count",
"label": null
}
}
8 changes: 8 additions & 0 deletions geonode/monitoring/admin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
RequestEvent,
ExceptionEvent,
MetricLabel,
MetricValue,
MonitoredResource,
NotificationCheck,
MetricNotificationCheck,
Expand Down Expand Up @@ -72,6 +73,13 @@ class MetricLabelAdmin(admin.ModelAdmin):
list_display = ('name',)


@admin.register(MetricValue)
class MetricValueAdmin(admin.ModelAdmin):
list_display = ('service_metric', 'service', 'event_type', 'resource', 'label',
'value', 'value_num', 'value_raw', 'samples_count', 'data')
list_filter = ('service_metric', 'service', 'event_type')


@admin.register(MonitoredResource)
class MonitoredResourceAdmin(admin.ModelAdmin):
list_display = ('id', 'name', 'type',)
Expand Down
34 changes: 33 additions & 1 deletion geonode/monitoring/collector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,19 @@ def get_metrics_data(self, metric_name,
'group_by': ['mr.id', 'mr.type', 'mr.name'],
'grouper': ['resource', 'name', 'type', 'id', 'resource_id'],
},
# resource count
'count_on_resource': {'select_only': [('count(distinct(mr.id)) as val, '
'count(1) as metric_count, '
'sum(samples_count) as samples_count, '
'sum(mv.value_num), min(mv.value_num), '
'max(mv.value_num)')],
'from': [('join monitoring_monitoredresource mr '
'on (mv.resource_id = mr.id)')],
'where': ['and mr.id is not NULL'],
'order_by': ['val desc'],
'group_by': [],
'grouper': [],
},
'event_type': {'select_only': ['ev.name as event_type', 'count(1) as val',
'count(1) as metric_count',
'sum(samples_count) as samples_count',
Expand All @@ -739,6 +752,7 @@ def get_metrics_data(self, metric_name,
'group_by': ['ev.name'],
'grouper': [],
},
# for each event the unique label count
'event_type_on_label': {'select_only': ['ev.name as event_type',
'count(distinct(ml.name)) as val',
'count(1) as metric_count',
Expand All @@ -753,6 +767,21 @@ def get_metrics_data(self, metric_name,
'group_by': ['ev.name'],
'grouper': [],
},
# for each event the unique user count
'event_type_on_user': {'select_only': ['ev.name as event_type',
'count(distinct(ml.user)) as val',
'count(1) as metric_count',
'sum(samples_count) as samples_count',
'sum(mv.value_num), min(mv.value_num)',
'max(mv.value_num)', ],
'from': ['join monitoring_eventtype ev on (ev.id = mv.event_type_id)',
('join monitoring_monitoredresource mr '
'on (mv.resource_id = mr.id)')],
'where': [],
'order_by': ['val desc'],
'group_by': ['ev.name'],
'grouper': [],
},
# group by user: number of unique user
'user': {'select_only': [('count(distinct(ml.user)) as val, '
'count(1) as metric_count, sum(samples_count) as samples_count, '
Expand Down Expand Up @@ -823,7 +852,10 @@ def get_metrics_data(self, metric_name,
'(ms.id = mv.service_id and ms.service_type_id = %(service_type_id)s ) ')
params['service_type_id'] = service_type.id

if group_by not in ('event_type', 'event_type_on_label',) and event_type is None:
if event_type is None and group_by not in (
'event_type',
'event_type_on_label',
'event_type_on_user'):
event_type = EventType.get(EventType.EVENT_ALL)

if event_type and metric_name not in BuiltIns.host_metrics:
Expand Down
432 changes: 417 additions & 15 deletions geonode/monitoring/tests/integration.py
View file

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions geonode/monitoring/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,15 @@ def event_type_type(val):
except EventType.DoesNotExist:
raise ValueError("Event Type {} doesn't exist".format(val))

@staticmethod
def ows_service_type(val):
if str(val).lower() in ("true", "1"):
return True
elif str(val).lower() in ("false", "0"):
return False
else:
raise ValueError("Invalid ows_service value {}".format(val))


def dump(obj, additional_fields=tuple()):
if hasattr(obj, '_meta'):
Expand Down
19 changes: 17 additions & 2 deletions geonode/monitoring/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,20 +159,23 @@ class MetricsFilters(CheckTypeForm):
GROUP_BY_CHOICES = ((GROUP_BY_RESOURCE, "By resource",),)
GROUP_BY_RESOURCE_ON_LABEL = 'resource_on_label'
GROUP_BY_RESOURCE_ON_USER = 'resource_on_user'
GROUP_BY_COUNT_ON_RESOURCE = 'count_on_resource'
GROUP_BY_LABEL = 'label'
GROUP_BY_USER = 'user'
GROUP_BY_USER_ON_LABEL = 'user_on_label'
GROUP_BY_EVENT_TYPE = 'event_type'
GROUP_BY_EVENT_TYPE_ON_LABEL = 'event_type_on_label'
GROUP_BY_EVENT_TYPE_ON_USER = 'event_type_on_user'
GROUP_BY_CHOICES = ((GROUP_BY_RESOURCE, "By resource",),
(GROUP_BY_RESOURCE_ON_LABEL, "By resource on label",),
(GROUP_BY_RESOURCE_ON_USER, "By resource on user",),
(GROUP_BY_COUNT_ON_RESOURCE, "By resource with count",),
(GROUP_BY_LABEL, "By label",),
(GROUP_BY_USER, "By user",),
(GROUP_BY_USER_ON_LABEL, "By user on label",),
(GROUP_BY_EVENT_TYPE, "By event type",),
(GROUP_BY_EVENT_TYPE_ON_LABEL, "By event type on label",),
)
(GROUP_BY_EVENT_TYPE_ON_USER, "By event type on user",),)
service = forms.CharField(required=False)
label = forms.CharField(required=False)
user = forms.CharField(required=False)
Expand Down Expand Up @@ -227,6 +230,13 @@ def clean_resource_type(self):
return self._check_type('resource_type')


class EventTypesFilterForm(CheckTypeForm):
ows_service = forms.CharField(required=False)

def clean_ows_service(self):
return self._check_type('ows_service')


class FilteredView(View):
# form which validates request.GET for get_queryset()
filter_form = None
Expand Down Expand Up @@ -363,11 +373,16 @@ def get_queryset(self, metric_name, valid_from,

@view_decorator(superuser_protected, subclass=True)
class EventTypeList(FilteredView):

filter_form = EventTypesFilterForm
fields_map = (('name', 'name',), ('type_label', 'type_label',),)
output_name = 'event_types'

def get_queryset(self, **kwargs):
if "ows_service" in kwargs and kwargs["ows_service"] is not None:
if kwargs["ows_service"]:
return EventType.objects.filter(name__icontains="OWS")
else:
return EventType.objects.exclude(name__icontains="OWS")
return EventType.objects.all()

def get(self, request, *args, **kwargs):
Expand Down