WIP: Metrics break down

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Current (in progress)
 
+### Breaking changes
+
+- Full metrics refactoring [2459](https://github.com/opendatateam/udata/pull/2459)
+
+### New features
+
 - Nothing yet
 
 ## 2.0.1 (2020-03-24)

docs/adapting-settings.md

@@ -640,12 +640,6 @@ The cache type, which can be adjusted to your needs (_ex:_ `null`, `memcached`)
 A prefix used for cache keys to avoid conflicts with other middleware.
 It also allows you to use the same backend with different instances.
 
-### USE_METRICS
-
-**default**: `True`
-
-This activates metrics, this is deactivated for tests
-
 ## Flask-FS options
 
 udata use Flask-FS as storage abstraction.

js/front/dashboard.js

@@ -7,10 +7,9 @@ import Vue from 'vue';
 
 import ActivityTimeline from 'components/activities/timeline.vue';
 import SmallBox from 'components/containers/small-box.vue';
-import DashboardGraphs from 'components/dashboard/graphs.vue';
 
 
 new Vue({
     mixins: [FrontMixin],
-    components: {ActivityTimeline, DashboardGraphs, SmallBox},
+    components: {ActivityTimeline, SmallBox},
 });

js/front/organization/index.js

@@ -14,7 +14,6 @@ import Vue from 'vue';
 import FollowButton from 'components/buttons/follow.vue';
 import DetailsModal from './details-modal.vue';
 import ActivityTimeline from 'components/activities/timeline.vue';
-import DashboardGraphs from 'components/dashboard/graphs.vue';
 import Tab from 'components/tab';
 import Tabset from 'components/tabs.vue';
 
@@ -25,7 +24,7 @@ import MembershipRequest from './membership-request.vue';
 
 new Vue({
     mixins: [FrontMixin],
-    components: {FollowButton, Tab, Tabset, ActivityTimeline, DashboardGraphs, SmallBox},
+    components: {FollowButton, Tab, Tabset, ActivityTimeline, SmallBox},
     data() {
         return {
             organization: this.extractOrganization(),

js/views/dataset.vue

@@ -35,10 +35,6 @@
     <div class="row">
         <resource-list :dataset="dataset" class="col-xs-12"></resource-list>
     </div>
-    <div class="row">
-        <chart-widget id="trafic" class="col-xs-12" :title="_('Audience')"
-            :metrics="metrics" x="date" :y="y"></chart-widget>
-    </div>
 
     <div class="row">
         <reuse-list id="reuses" class="col-xs-12" :reuses="reuses"></reuse-list>

js/views/organization.vue

@@ -13,15 +13,6 @@
         <members :org="org" class="col-xs-12 col-md-6"></members>
     </div>
 
-    <div class="row">
-        <chart id="trafic-widget" class="col-xs-12"
-            :title="charts.traffic.title" :default="charts.traffic.default"
-            :metrics="metrics"
-            x="date" :y="charts.traffic.y"
-            >
-        </chart>
-    </div>
-
     <div class="row">
         <dataset-list id="datasets-widget" class="col-xs-12" :datasets="datasets"
             :downloads="downloads">

udata/core/dataset/api_fields.py

@@ -69,7 +69,8 @@
     'last_modified': fields.ISODateTime(
         attribute='modified', readonly=True,
         description='The resource last modification date'),
-    'metrics': fields.Raw(description='The resource metrics', readonly=True),
+    'metrics': fields.Raw(
+        description='The resource metrics', readonly=True),
     'extras': fields.Raw(description='Extra attributes as key-value pairs'),
     'preview_url': fields.String(description='An optional preview URL to be '
                                  'loaded as a standalone page (ie. iframe or '
@@ -161,7 +162,7 @@
         description=('Next expected update date, you will be notified '
                      'once that date is reached.')),
     'extras': fields.Raw(description='Extras attributes as key-value pairs'),
-    'metrics': fields.Raw(description='The dataset metrics'),
+    'metrics': fields.Raw(attribute=lambda o: o.get_metrics(), description='The dataset metrics'),
     'organization': fields.Nested(
         org_ref_fields, allow_null=True,
         description='The producer organization'),

udata/core/dataset/metrics.py

@@ -1,42 +1,9 @@
-from udata.core.metrics import Metric
-from udata.i18n import lazy_gettext as _
-from udata.models import Dataset, Reuse
+from udata.models import Reuse
 
-from udata.core.issues.metrics import IssuesMetric
-from udata.core.discussions.metrics import DiscussionsMetric
-from udata.core.followers.metrics import FollowersMetric
 
-
-__all__ = (
-    'DatasetReuses', 'DatasetIssuesMetric', 'DatasetFollowers',
-    'DatasetDiscussionsMetric'
-)
-
-
-class DatasetReuses(Metric):
-    model = Dataset
-    name = 'reuses'
-    display_name = _('Reuses')
-
-    def get_value(self):
-        return Reuse.objects(datasets=self.target).visible().count()
-
-
-@Reuse.on_update.connect
 @Reuse.on_create.connect
+@Reuse.on_update.connect
+@Reuse.on_delete.connect
 def update_dataset_reuses_metric(reuse, **kwargs):
     for dataset in reuse.datasets:
-        metric = DatasetReuses(dataset)
-        metric.trigger_update()
-
-
-class DatasetIssuesMetric(IssuesMetric):
-    model = Dataset
-
-
-class DatasetDiscussionsMetric(DiscussionsMetric):
-    model = Dataset
-
-
-class DatasetFollowers(FollowersMetric):
-    model = Dataset
+        dataset.count_reuses()

udata/core/dataset/models.py

@@ -8,6 +8,7 @@
 from mongoengine.fields import DateTimeField
 from stringdist import rdlevenshtein
 from werkzeug import cached_property
+from elasticsearch_dsl import Integer, Object
 
 from udata.frontend.markdown import mdstrip
 from udata.models import db, WithMetrics, BadgeMixin, SpatialCoverage
@@ -394,6 +395,20 @@ def __str__(self):
         PIVOTAL_DATA: _('Pivotal data'),
     }
 
+    __search_metrics__ = Object(properties={
+        'reuses': Integer(),
+        'followers': Integer(),
+        'views': Integer(),
+    })
+
+    __metrics_keys__ = [
+        'discussions',
+        'issues',
+        'reuses',
+        'followers',
+        'views',
+    ]
+
     meta = {
         'indexes': [
             '-created_at',
@@ -439,7 +454,6 @@ def post_save(cls, sender, document, **kwargs):
                                        resource_id=kwargs['resource_added'])
 
     def clean(self):
-        super(Dataset, self).clean()
         if self.frequency in LEGACY_FREQUENCIES:
             self.frequency = LEGACY_FREQUENCIES[self.frequency]
 
@@ -679,6 +693,30 @@ def json_ld(self):
 
         return result
 
+    @property
+    def views_count(self):
+        return self.metrics.get('views', 0)
+
+    def count_discussions(self):
+        from udata.models import Discussion
+        self.metrics['discussions'] = Discussion.objects(subject=self, closed=None).count()
+        self.save()
+
+    def count_issues(self):
+        from udata.models import Issue
+        self.metrics['issues'] = Issue.objects(subject=self, closed=None).count()
+        self.save()
+
+    def count_reuses(self):
+        from udata.models import Reuse
+        self.metrics['reuses'] = Reuse.objects(datasets=self).visible().count()
+        self.save()
+
+    def count_followers(self):
+        from udata.models import Follow
+        self.metrics['followers'] = Follow.objects(until=None).followers(self).count()
+        self.save()
+
 
 pre_save.connect(Dataset.pre_save, sender=Dataset)
 post_save.connect(Dataset.post_save, sender=Dataset)

udata/core/dataset/search.py

@@ -11,7 +11,7 @@
     Dataset, Organization, License, User, GeoZone, RESOURCE_TYPES
 )
 from udata.search import (
-    ModelSearchAdapter, i18n_analyzer, metrics_mapping_for, register,
+    ModelSearchAdapter, i18n_analyzer, register,
     lazy_config
 )
 from udata.search.analysis import simple
@@ -21,8 +21,6 @@
 )
 from udata.utils import to_iso_datetime
 
-# Metrics are require for dataset search
-from . import metrics  # noqa
 
 __all__ = ('DatasetSearch', )
 
@@ -33,11 +31,11 @@
 
 
 def max_reuses():
-    return max(current_site.metrics.get('max_dataset_reuses'), 10)
+    return max(current_site.get_metrics()['max_dataset_reuses'], 10)
 
 
 def max_followers():
-    return max(current_site.metrics.get('max_dataset_followers'), 10)
+    return max(current_site.get_metrics()['max_dataset_followers'], 10)
 
 
 def granularity_labelizer(value):
@@ -98,7 +96,7 @@ class Meta:
                               payloads=False)
     created = Date(format='date_hour_minute_second')
     last_modified = Date(format='date_hour_minute_second')
-    metrics = metrics_mapping_for(Dataset)
+    metrics = Dataset.__search_metrics__
     featured = Boolean()
     temporal_coverage = Nested(multi=False, properties={
         'start': Long(),

udata/core/dataset/tasks.py

@@ -13,7 +13,7 @@
 from udata.frontend import csv
 from udata.harvest.models import HarvestJob
 from udata.i18n import lazy_gettext as _
-from udata.models import (Follow, Issue, Discussion, Activity, Metrics, Topic,
+from udata.models import (Follow, Issue, Discussion, Activity, Topic,
                           Organization)
 from udata.tasks import job
 
@@ -42,8 +42,6 @@ def purge_datasets(self):
         Discussion.objects(subject=dataset).delete()
         # Remove activity
         Activity.objects(related_to=dataset).delete()
-        # Remove metrics
-        Metrics.objects(object_id=dataset.id).delete()
         # Remove topics' related dataset
         for topic in Topic.objects(datasets=dataset):
             datasets = topic.datasets