calcule aggregation data

app/controller/reportsAggregation.py

@@ -1,7 +1,6 @@
-import pandas as pd
-from flask import request
-from flask_restful import Resource
 
+from flask_restful import Resource
+from app.libs.makeAggregation import make_aggregation
 from app.repository.reports import Reports
 
 
@@ -24,15 +23,9 @@ def get(self, table_name):
         """
         Report = Reports(table_name)
         data = Report.getAll(limit=99999)
+        aggr = make_aggregation(data)
 
-        df =  pd.DataFrame(data)
-
-        datacenters = df['datacenters'].apply(pd.Series)
-
-
-        print(datacenters.groupby('name', as_index=False).agg({"_id": "count"}))
-
-
-
-        req = request.args.to_dict()
-        return data
+        return {
+            'name': table_name,
+            'aggr': aggr
+        }

app/libs/makeAggregation.py

@@ -0,0 +1,23 @@
+
+import pandas as pd
+from app.services.aggregator.factoryAggr import FactoryAggr
+from app.services.mappers.aggregator import mapperA
+
+
+def make_aggregation(data):
+    aggr = {}
+
+    df = pd.DataFrame(data)
+    factory = FactoryAggr(df)
+
+    for mapp in mapperA():
+        result = factory.run(mapp)
+
+        if result is not None:
+            entity = mapp.uniqueField()
+            aggr[entity] = {
+                'label': result.index.tolist(),
+                'data': result.tolist()
+            }
+
+    return aggr

app/services/aggregator/aggr.py

@@ -0,0 +1,16 @@
+
+class Aggr(object):
+
+    def __init__(self, field, sub=''):
+        self._field = field
+        self._sub = sub
+        self._result = []
+
+    def getField(self):
+        return self._field
+
+    def uniqueField(self):
+        return "%s_%s" % (self._field, self._sub)
+
+    def getResult(self):
+        return self._result

app/services/aggregator/aggrDict.py

@@ -0,0 +1,16 @@
+import pandas as pd
+from.aggr import Aggr
+
+class AggrDict(Aggr):
+
+    def __init__(self, field,  sub="name", aggrk="_id", aggr="count"):
+        self._aggr = aggr
+        self._aggrk = aggrk
+        super().__init__(field, sub)
+
+    def aggregate(self, df):
+        dic = df.dropna().apply(pd.Series)
+
+        if self._sub in dic:
+            aggr = dic.groupby(self._sub).agg({self._aggrk: self._aggr})
+            self._result = aggr.get(self._aggrk)

app/services/aggregator/aggrImmutable.py

@@ -0,0 +1,7 @@
+
+from.aggr import Aggr
+
+class AggrImmutable(Aggr):
+
+    def aggregate(self, df):
+        self._result = df.groupby(df).count()

app/services/aggregator/aggrListObj.py

@@ -0,0 +1,29 @@
+
+import pandas as pd
+from.aggr import Aggr
+
+class AggrListObj(Aggr):
+
+    def aggregate(self, df):
+
+        dic = df\
+            .dropna()\
+            .apply(self.merged)\
+            .stack()\
+            .reset_index(level=1, drop=True)
+
+        self._result = dic.groupby(dic).count()
+
+    def merged(self, services):
+
+        if services and isinstance(services, list):
+            services = map(self.reducev, services)
+
+        return pd.Series(services)
+
+    def reducev(self, data):
+        if isinstance(data, dict):
+            return data.get(self._sub)
+
+        if isinstance(data, (str, int, float)):
+            return data

app/services/aggregator/factoryAggr.py

@@ -0,0 +1,15 @@
+
+
+class FactoryAggr(object):
+
+    def __init__(self, dataframe):
+        self._dataframe = dataframe
+
+    def run(self, cls):
+        field = cls.getField()
+
+        if field in self._dataframe:
+            df = self._dataframe[field]
+            cls.aggregate(df)
+
+            return cls.getResult()

app/services/mappers/aggregator.py

@@ -0,0 +1,28 @@
+
+from app.services.aggregator.aggrDict import AggrDict
+from app.services.aggregator.aggrListObj import AggrListObj
+from app.services.aggregator.aggrImmutable import AggrImmutable
+
+def mapperA():
+    return [
+        AggrImmutable("family"),
+        AggrImmutable("size"),
+        AggrImmutable("provider"),
+        AggrDict("datacenters", sub="name"),
+        AggrDict("datacenters", sub="provider"),
+        AggrDict("datacenters", sub="instance"),
+        AggrDict("datacenters", sub="region"),
+        AggrDict("os", sub="base", aggrk="base"),
+        AggrListObj("regions"),
+        AggrListObj("zones"),
+        AggrListObj("deps", sub="name"),
+        AggrListObj("services", sub="name"),
+        AggrListObj("deploy", sub="type"),
+        AggrListObj("contacts", sub="channel"),
+        AggrListObj("applications", sub="name"),
+        AggrListObj("system", sub="name"),
+        AggrListObj("clients", sub="name"),
+        AggrListObj("entry", sub="name"),
+        AggrListObj("auth", sub="type"),
+        AggrListObj("tags", sub="key"),
+    ]

app/tasks/upload_json.py

@@ -5,6 +5,7 @@
 from app.tasks.webhook import task_webhook
 from app.tasks.notification import task_notification
 from app.tasks.ws import task_ws
+from app.libs.makeAggregation import make_aggregation
 
 
 def batch(iterable, n=1):
@@ -25,8 +26,10 @@ def task_upload(report_id, owner_user, name, result):
         webhook_id.append(str(tt))
 
     prefetch = DataFrame(result[:50], False).getHeaders()
+    aggr = make_aggregation(result)
+
     notification_id = task_notification.delay(report_id=report_id, msg=id, status='finished',
-                                              more={'columns': prefetch})
+                                              more={'columns': prefetch, 'aggr': aggr})
     task_ws.delay(name, report_id, owner_user)