Fixed bugs in sankey and validated request args output

distill/algorithms/graphs/graph.py

@@ -31,7 +31,8 @@ class GraphAnalytics (object):
     def generate_graph(app,
                        app_type='logs',
                        log_type='raw',
-                       target_events=[],
+                       targets=[],
+                       events=[],
                        time_range=['now-1h', 'now'],
                        size=20):
         """
@@ -62,15 +63,41 @@ def generate_graph(app,
         ]
 
         # Filtering
-        # should_query = []
-        if (target_events):
-            for event in target_events:
-                res = {
-                    "term": {
-                        "type": event
-                    }
+        should_query = []
+        must_query = []
+
+        # Include these events in the request
+        if events:
+            include_events = {
+                "terms": {
+                    "type.keyword": events
+                }
+            }
+            filter_query.append(include_events)
+
+        target_in = targets[0]
+        target_out = targets[1]
+
+        if target_in:
+            include_targets = {
+                "terms": {
+                    "target.keyword": target_in
                 }
-                filter_query.append(res)
+            }
+
+            filter_query.append(include_targets)
+
+        # Remove these elementids from result set
+        for target in target_out:
+            res = {
+                "term": {
+                    "target.keyword": target
+                }
+            }
+            must_not_query.append(res)
+
+        # Finish off should query
+        # must_query.append({"bool": {"should": should_query}})
 
         # Sort By Time
         sort_query = [
@@ -98,7 +125,7 @@ def generate_graph(app,
         session_query = {
                 "terms": {
                     "field": "sessionID.keyword",
-                    "min_doc_count": 1
+                    # "min_doc_count": 1
                 }
             }
 
@@ -108,14 +135,14 @@ def generate_graph(app,
         target_query = {
                 "terms": {
                     "field": "target.keyword",
-                    "min_doc_count": 1,
+                    # "min_doc_count": 1,
                     "size": size
                 },
                 "aggs": {
                     "events": {
                         "terms": {
                             "field": "type.keyword",
-                            "min_doc_count": 1,
+                            # "min_doc_count": 1,
                             "size": size
                         }
                     },
@@ -142,10 +169,11 @@ def generate_graph(app,
             "sort": sort_query,
             "query": {
                 "bool": {
-                    # "must": must_match,
+                    # "must": must_query,
                     # "should": should_query,
                     "filter": filter_query,
                     "must_not": must_not_query,
+                    # "minimum_should_match": len(should_query) - 1
                 }
             },
             "_source": {
@@ -165,7 +193,6 @@ def generate_graph(app,
         # return query
         # Process Aggregate Results
         response = es.search(app, doc_type=app_type, body=query, size=0)
-
         # Only want to look at aggregations
         sessions = response['aggregations']['sessions']['buckets']
         # allSessions = { x['key']: [] for x in sessions }
@@ -285,10 +312,10 @@ def generate_graph(app,
                     nodename2 = node2['target']
 
                     seqID = '%s->%s' % (nodename1, nodename2)
-                    print(seqID)
+                    #print(seqID)
 
                     if nodename1 != nodename2:  #double check again for self-loops
-                        print(node1)
+                        #print(node1)
                         link = {
                             'sequenceID': seqID,
                             'sourceName': nodename1,

distill/app.py

@@ -83,26 +83,37 @@ def sankey(app_id):
     # Time range using date math
     from_range = 'now-15m'
     to_range = 'now'
-    ts_range = [from_range, to_range]
-    if 'from' in request.args:
+
+    if 'from' in request.args and request.args.get('from') != '':
         from_range = request.args.get('from')
 
-        if 'to' in request.args:
-            to_range = request.args.get('to')
-            ts_range = [from_range, to_range]
+    if 'to' in request.args and request.args.get('to') != '':
+        to_range = request.args.get('to')
+
+    ts_range = [from_range, to_range]
 
     # Size
     size = 20
-    if 'size' in request.args:
+    if 'size' in request.args and request.args.get('size') != '':
         size = request.args.get('size')
 
-    # target events
+    # events (event_in)
     events = []
-    if 'event' in request.args:
-        events.append(request.args.get('event'))
+    if 'event' in request.args and request.args.get('event') != '':
+        events = request.args.get('event').split(',')
+
+    # filter in/out targets
+    target_in = []
+    if 'target_in' in request.args and request.args.get('target_in') != '':
+        target_in = request.args.get('target_in').split(',')
+
+    target_out = []
+    if 'target_out' in request.args and request.args.get('target_out') != '':
+        target_out = request.args.get('target_out').split(',')
 
     return jsonify(GraphAnalytics.generate_graph(app_id,
-                                                 target_events=events,
+                                                 targets=[target_in, target_out],
+                                                 events=events,
                                                  time_range=ts_range,
                                                  size=size))