Merge pull request #931 from neuroscout/ext_count

add distinct extractor route

neuroscout/core.py

@@ -66,7 +66,8 @@
         ('TaskPredictorsResource', 'tasks/<int:task_id>/predictors'),
         ('TaskListResource', 'tasks'),
         ('ImageVersionResource', 'image_version'),
-        ('ExtractorListResource', 'extractors')
+        ('ExtractorListResource', 'extractors'),
+        ('ExtractorDistinctResource', 'extractors/distinct')
     ])
 
 

neuroscout/resources/__init__.py

@@ -7,7 +7,7 @@
                        AnalysisFillResource, AnalysisUploadResource,
                        BibliographyResource, ImageVersionResource)
 from .dataset import DatasetResource, DatasetListResource
-from .extractor import ExtractorListResource
+from .extractor import ExtractorListResource, ExtractorDistinctResource
 from .predictor import (PredictorListResource, PredictorResource,
                         PredictorCollectionCreateResource,
                         TaskPredictorsResource,
@@ -36,6 +36,7 @@
     'DatasetResource',
     'DatasetListResource',
     'ExtractorListResource',
+    'ExtractorDistinctResource',
     'PredictorResource',
     'PredictorListResource',
     'PredictorEventListResource',

neuroscout/resources/extractor.py

@@ -1,11 +1,12 @@
 import json
 import re
+from webargs import fields
 from pliers import extractors as piext
-from flask_apispec import MethodResource, marshal_with, doc
+from flask_apispec import MethodResource, use_kwargs, marshal_with, doc
 from flask import current_app
 from ..core import cache
 from ..schemas.extractor import ExtractorSchema
-
+from ..utils.misc import distinct_extractors
 
 class ExtractorListResource(MethodResource):
     @doc(tags=['extractor'], summary='Extractor descriptions')
@@ -31,3 +32,22 @@ def get(self, **kwargs):
                 pass
 
         return result
+
+
+class ExtractorDistinctResource(MethodResource):
+    @doc(tags=['extractor'], summary='Extractor counts by dataset/task')
+    @cache.cached(60 * 60 * 24 * 300, query_string=True)
+    @use_kwargs({
+        'count': fields.Boolean(
+            missing=True,
+            description="Return counts, or distinct names"),
+        'active_only': fields.Boolean(
+            missing=True,
+            description="Return results only for active Datasets")
+        },
+        location='query')
+    def get(self, **kwargs):
+        count = kwargs.pop('count')
+        active = kwargs.pop('active_only')
+
+        return distinct_extractors(count=count, active=active)

neuroscout/utils/misc.py

@@ -0,0 +1,33 @@
+""" Miscelanous tools """
+
+from .. import models as ms
+
+
+def distinct_extractors(count=True, active=True):
+    """ Tool to count unique number of predictors for each Dataset/Task """
+    active_datasets = ms.Dataset.query.filter_by(active=active)
+    superset = set([v for (v, ) in ms.Predictor.query.filter_by(active=True).filter(
+        ms.Predictor.dataset_id.in_(
+            active_datasets.with_entities('id'))).join(
+                ms.ExtractedFeature).distinct(
+                    'extractor_name').values('extractor_name')])
+
+    res = {}
+
+    for en in superset:
+        for ds in active_datasets:
+            for t in ds.tasks:
+                name = f"{ds.name}_{t.name}"
+                if name not in res:
+                    res[name] = {}
+                preds = ms.Predictor.query.filter_by(
+                    dataset_id=ds.id, active=True).join(
+                        ms.ExtractedFeature).filter_by(
+                            extractor_name=en).distinct('feature_name')
+                if count:
+                    r = preds.count()
+                else:
+                    r = list(preds.values('name'))
+                res[name][en] = r
+
+    return res