Add harvesters count to site metrics

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Current (in progress)
 
+- Add harvesters count to site metrics [#2890](https://github.com/opendatateam/udata/pull/2890)
 - Return 400 instead of 500 in case of not ObjectID arg in API [#2889](https://github.com/opendatateam/udata/pull/2889)
 - Use a single session for reindex [#2891](https://github.com/opendatateam/udata/pull/2891)
 

udata/core/metrics/commands.py

@@ -4,7 +4,7 @@
 
 from flask import current_app
 
-from udata.commands import cli, success, echo, white
+from udata.commands import cli, success
 from udata.models import User, Dataset, Reuse, Organization, Site
 
 log = logging.getLogger(__name__)
@@ -43,6 +43,7 @@ def update(site=False, organizations=False, users=False, datasets=False,
             site.count_reuses()
             site.count_followers()
             site.count_discussions()
+            site.count_harvesters()
             site.count_max_dataset_followers()
             site.count_max_dataset_reuses()
             site.count_max_reuse_datasets()
@@ -53,7 +54,6 @@ def update(site=False, organizations=False, users=False, datasets=False,
         except Exception as e:
             log.info(f'Error during update: {e}')
 
-
     if do_all or datasets:
         log.info('Update datasets metrics')
         all_datasets = Dataset.objects.visible().timeout(False)

udata/core/metrics/tasks.py

@@ -4,6 +4,7 @@
 from udata.tasks import job
 from udata.core.metrics.signals import on_site_metrics_computed
 
+
 @job('compute-site-metrics')
 def compute_site_metrics(self):
     site = Site.objects(id=current_app.config['SITE_ID']).first()
@@ -14,6 +15,7 @@ def compute_site_metrics(self):
     site.count_reuses()
     site.count_followers()
     site.count_discussions()
+    site.count_harvesters()
     site.count_max_dataset_followers()
     site.count_max_dataset_reuses()
     site.count_max_reuse_datasets()

udata/core/site/models.py

@@ -43,6 +43,7 @@ class Site(WithMetrics, db.Document):
         'resources',
         'reuses',
         'users',
+        'harvesters'
     ]
 
     def __str__(self):
@@ -71,7 +72,7 @@ def count_datasets(self):
     def count_resources(self):
         self.metrics['resources'] = next(Dataset.objects.visible().aggregate(
             {'$project': {'resources': 1}},
-            {'$unwind': '$resources' },
+            {'$unwind': '$resources'},
             {'$group': {'_id': 'result', 'count': {'$sum': 1}}}
         ), {}).get('count', 0)
         self.save()
@@ -90,6 +91,11 @@ def count_discussions(self):
         self.metrics['discussions'] = Discussion.objects.count()
         self.save()
 
+    def count_harvesters(self):
+        from udata.harvest.models import HarvestSource
+        self.metrics['harvesters'] = HarvestSource.objects().count()
+        self.save()
+
     def count_max_dataset_followers(self):
         dataset = (Dataset.objects(metrics__followers__gt=0).visible()
                           .order_by('-metrics.followers').first())
@@ -98,7 +104,7 @@ def count_max_dataset_followers(self):
 
     def count_max_dataset_reuses(self):
         dataset = (Dataset.objects(metrics__reuses__gt=0).visible()
-                        .order_by('-metrics.reuses').first())
+                   .order_by('-metrics.reuses').first())
         self.metrics['max_dataset_reuses'] = dataset.metrics['reuses'] if dataset else 0
         self.save()
 

udata/tests/site/test_site_metrics.py

@@ -2,10 +2,9 @@
 
 from udata.core.dataset.factories import DatasetFactory, VisibleDatasetFactory, OrganizationFactory
 from udata.core.reuse.factories import VisibleReuseFactory
+from udata.harvest.tests.factories import HarvestSourceFactory
 from udata.core.site.factories import SiteFactory
-from udata.models import Site, Badge, PUBLIC_SERVICE
-from udata.core.site.models import current_site
-from udata.tests.helpers import assert_emit
+from udata.models import Badge, PUBLIC_SERVICE
 
 
 @pytest.mark.usefixtures('clean_db')
@@ -68,3 +67,13 @@ def test_badges_metric(self, app):
         site.count_org_for_badge(PUBLIC_SERVICE)
 
         assert site.get_metrics()[PUBLIC_SERVICE] == len(public_services)
+
+    def test_harvesters_metric(self, app):
+        site = SiteFactory.create(
+            id=app.config['SITE_ID']
+        )
+        sources = [HarvestSourceFactory() for i in range(10)]
+
+        site.count_harvesters()
+
+        assert site.get_metrics()['harvesters'] == len(sources)