Merge pull request #11928 from fosterseth/job_event_children_summary

Add JobJobEventsChildrenSummary endpoint

awx/api/templates/api/job_job_events_children_summary.md

@@ -0,0 +1,102 @@
+# View a summary of children events
+
+Special view to facilitate processing job output in the UI.
+In order to collapse events and their children, the UI needs to know how
+many children exist for a given event.
+The UI also needs to know the order of the event (0 based index), which
+usually matches the counter, but not always.
+This view returns a JSON object where the key is the event counter, and the value
+includes the number of children (and grandchildren) events.
+Only events with children are included in the output.
+
+## Example
+
+e.g. Demo Job Template job
+tuple(event counter, uuid, parent_uuid)
+
+```
+(1, '4598d19e-93b4-4e33-a0ae-b387a7348964', '')
+(2, 'aae0d189-e3cb-102a-9f00-000000000006', '4598d19e-93b4-4e33-a0ae-b387a7348964')
+(3, 'aae0d189-e3cb-102a-9f00-00000000000c', 'aae0d189-e3cb-102a-9f00-000000000006')
+(4, 'f4194f14-e406-4124-8519-0fdb08b18f4b', 'aae0d189-e3cb-102a-9f00-00000000000c')
+(5, '39f7ad99-dbf3-41e0-93f8-9999db4004f2', 'aae0d189-e3cb-102a-9f00-00000000000c')
+(6, 'aae0d189-e3cb-102a-9f00-000000000008', 'aae0d189-e3cb-102a-9f00-000000000006')
+(7, '39a49992-5ca4-4b6c-b178-e56d0b0333da', 'aae0d189-e3cb-102a-9f00-000000000008')
+(8, '504f3b28-3ea8-4f6f-bd82-60cf8e807cc0', 'aae0d189-e3cb-102a-9f00-000000000008')
+(9, 'a242be54-ebe6-4021-afab-f2878bff2e9f', '4598d19e-93b4-4e33-a0ae-b387a7348964')
+```
+
+output
+
+```
+{
+"1": {
+    "rowNumber": 0,
+    "numChildren": 8
+},
+"2": {
+    "rowNumber": 1,
+    "numChildren": 6
+},
+"3": {
+    "rowNumber": 2,
+    "numChildren": 2
+},
+"6": {
+    "rowNumber": 5,
+    "numChildren": 2
+}
+}
+"meta_event_nested_parent_uuid": {}
+}
+```
+
+counter 1 is event 0, and has 8 children
+counter 2 is event 1, and has 6 children
+etc.
+
+The UI also needs to be able to collapse over "meta" events -- events that
+show up due to verbosity or warnings from the system while the play is running.
+These events have a 0 level event, with no parent uuid.
+
+```
+playbook_on_start
+verbose
+  playbook_on_play_start
+    playbook_on_task_start
+      runner_on_start        <- level 3
+verbose                      <- jump to level 0
+verbose
+        runner_on_ok         <- jump back to level 3
+      playbook_on_task_start
+        runner_on_start
+        runner_on_ok
+verbose
+verbose
+      playbook_on_stats
+```
+
+These verbose statements that fall in the middle of a series of children events
+are problematic for the UI.
+To help, this view will attempt to place the events into the hierarchy, without
+the event level jumps.
+
+```
+playbook_on_start
+  verbose
+  playbook_on_play_start
+    playbook_on_task_start
+      runner_on_start        <- A
+        verbose              <- this maps to the uuid of A
+        verbose
+        runner_on_ok
+      playbook_on_task_start <- B
+        runner_on_start
+        runner_on_ok
+        verbose              <- this maps to the uuid of B
+        verbose
+      playbook_on_stats
+```
+
+The output will include a JSON object where the key is the event counter,
+and the value is the assigned nested uuid.

awx/api/urls/job.py

@@ -10,6 +10,7 @@
     JobRelaunch,
     JobCreateSchedule,
     JobJobHostSummariesList,
+    JobJobEventsChildrenSummary,
     JobJobEventsList,
     JobActivityStreamList,
     JobStdout,
@@ -27,6 +28,7 @@
     re_path(r'^(?P<pk>[0-9]+)/create_schedule/$', JobCreateSchedule.as_view(), name='job_create_schedule'),
     re_path(r'^(?P<pk>[0-9]+)/job_host_summaries/$', JobJobHostSummariesList.as_view(), name='job_job_host_summaries_list'),
     re_path(r'^(?P<pk>[0-9]+)/job_events/$', JobJobEventsList.as_view(), name='job_job_events_list'),
+    re_path(r'^(?P<pk>[0-9]+)/job_events/children_summary/$', JobJobEventsChildrenSummary.as_view(), name='job_job_events_children_summary'),
     re_path(r'^(?P<pk>[0-9]+)/activity_stream/$', JobActivityStreamList.as_view(), name='job_activity_stream_list'),
     re_path(r'^(?P<pk>[0-9]+)/stdout/$', JobStdout.as_view(), name='job_stdout'),
     re_path(r'^(?P<pk>[0-9]+)/notifications/$', JobNotificationsList.as_view(), name='job_notifications_list'),

awx/api/views/__init__.py

@@ -3842,6 +3842,84 @@ def get_queryset(self):
         return job.get_event_queryset().select_related('host').order_by('start_line')
 
 
+class JobJobEventsChildrenSummary(APIView):
+
+    renderer_classes = [JSONRenderer]
+    meta_events = ('debug', 'verbose', 'warning', 'error', 'system_warning', 'deprecated')
+
+    def get(self, request, **kwargs):
+        resp = dict(children_summary={}, meta_event_nested_uuid={}, event_processing_finished=False)
+        job = get_object_or_404(models.Job, pk=kwargs['pk'])
+        if not job.event_processing_finished:
+            return Response(resp)
+        else:
+            resp["event_processing_finished"] = True
+
+        events = list(job.get_event_queryset().values('counter', 'uuid', 'parent_uuid', 'event').order_by('counter'))
+        if len(events) == 0:
+            return Response(resp)
+
+        # key is counter, value is number of total children (including children of children, etc.)
+        map_counter_children_tally = {i['counter']: {"rowNumber": 0, "numChildren": 0} for i in events}
+        # key is uuid, value is counter
+        map_uuid_counter = {i['uuid']: i['counter'] for i in events}
+        # key is uuid, value is parent uuid. Used as a quick lookup
+        map_uuid_puuid = {i['uuid']: i['parent_uuid'] for i in events}
+        # key is counter of meta events (i.e. verbose), value is uuid of the assigned parent
+        map_meta_counter_nested_uuid = {}
+
+        prev_non_meta_event = events[0]
+        for i, e in enumerate(events):
+            if not e['event'] in JobJobEventsChildrenSummary.meta_events:
+                prev_non_meta_event = e
+            if not e['uuid']:
+                continue
+            puuid = e['parent_uuid']
+
+            # if event is verbose (or debug, etc), we need to "assign" it a
+            # parent. This code looks at the event level of the previous
+            # non-verbose event, and the level of the next (by looking ahead)
+            # non-verbose event. The verbose event is assigned the same parent
+            # uuid of the higher level event.
+            # e.g.
+            # E1
+            #  E2
+            # verbose
+            # verbose <- we are on this event currently
+            #    E4
+            # We'll compare E2 and E4, and the verbose event
+            # will be assigned the parent uuid of E4 (higher event level)
+            if e['event'] in JobJobEventsChildrenSummary.meta_events:
+                event_level_before = models.JobEvent.LEVEL_FOR_EVENT[prev_non_meta_event['event']]
+                # find next non meta event
+                z = i
+                next_non_meta_event = events[-1]
+                while z < len(events):
+                    if events[z]['event'] not in JobJobEventsChildrenSummary.meta_events:
+                        next_non_meta_event = events[z]
+                        break
+                    z += 1
+                event_level_after = models.JobEvent.LEVEL_FOR_EVENT[next_non_meta_event['event']]
+                if event_level_after and event_level_after > event_level_before:
+                    puuid = next_non_meta_event['parent_uuid']
+                else:
+                    puuid = prev_non_meta_event['parent_uuid']
+                if puuid:
+                    map_meta_counter_nested_uuid[e['counter']] = puuid
+            map_counter_children_tally[e['counter']]['rowNumber'] = i
+            if not puuid:
+                continue
+            # now traverse up the parent, grandparent, etc. events and tally those
+            while puuid:
+                map_counter_children_tally[map_uuid_counter[puuid]]['numChildren'] += 1
+                puuid = map_uuid_puuid.get(puuid, None)
+
+        # create new dictionary, dropping events with 0 children
+        resp["children_summary"] = {k: v for k, v in map_counter_children_tally.items() if v['numChildren'] != 0}
+        resp["meta_event_nested_uuid"] = map_meta_counter_nested_uuid
+        return Response(resp)
+
+
 class AdHocCommandList(ListCreateAPIView):
 
     model = models.AdHocCommand

awx/main/tests/functional/api/test_events.py

@@ -46,3 +46,41 @@ def test_ad_hoc_events_sublist_truncation(get, organization_factory, job_templat
 
     response = get(url, user=objs.superusers.admin, expect=200)
     assert (len(response.data['results'][0]['stdout']) == 1025) == expected
+
+
+@pytest.mark.django_db
+def test_job_job_events_children_summary(get, organization_factory, job_template_factory):
+    objs = organization_factory("org", superusers=['admin'])
+    jt = job_template_factory("jt", organization=objs.organization, inventory='test_inv', project='test_proj').job_template
+    job = jt.create_unified_job()
+    url = reverse('api:job_job_events_children_summary', kwargs={'pk': job.pk})
+    response = get(url, user=objs.superusers.admin, expect=200)
+    assert response.data["event_processing_finished"] == False
+    '''
+    E1
+      E2
+        E3
+        E4 (verbose)
+      E5
+    '''
+    JobEvent.create_from_data(
+        job_id=job.pk, uuid='uuid1', parent_uuid='', event="playbook_on_start", counter=1, stdout='a' * 1024, job_created=job.created
+    ).save()
+    JobEvent.create_from_data(
+        job_id=job.pk, uuid='uuid2', parent_uuid='uuid1', event="playbook_on_play_start", counter=2, stdout='a' * 1024, job_created=job.created
+    ).save()
+    JobEvent.create_from_data(
+        job_id=job.pk, uuid='uuid3', parent_uuid='uuid2', event="runner_on_start", counter=3, stdout='a' * 1024, job_created=job.created
+    ).save()
+    JobEvent.create_from_data(job_id=job.pk, uuid='uuid4', parent_uuid='', event='verbose', counter=4, stdout='a' * 1024, job_created=job.created).save()
+    JobEvent.create_from_data(
+        job_id=job.pk, uuid='uuid5', parent_uuid='uuid1', event="playbook_on_task_start", counter=5, stdout='a' * 1024, job_created=job.created
+    ).save()
+    job.emitted_events = job.get_event_queryset().count()
+    job.status = "successful"
+    job.save()
+    url = reverse('api:job_job_events_children_summary', kwargs={'pk': job.pk})
+    response = get(url, user=objs.superusers.admin, expect=200)
+    assert response.data["children_summary"] == {1: {"rowNumber": 0, "numChildren": 4}, 2: {"rowNumber": 1, "numChildren": 2}}
+    assert response.data["meta_event_nested_uuid"] == {4: "uuid2"}
+    assert response.data["event_processing_finished"] == True