diff --git a/awx/api/templates/api/job_job_events_children_summary.md b/awx/api/templates/api/job_job_events_children_summary.md new file mode 100644 index 000000000000..7ad34ac429be --- /dev/null +++ b/awx/api/templates/api/job_job_events_children_summary.md @@ -0,0 +1,102 @@ +# View a summary of children events + +Special view to facilitate processing job output in the UI. +In order to collapse events and their children, the UI needs to know how +many children exist for a given event. +The UI also needs to know the order of the event (0 based index), which +usually matches the counter, but not always. +This view returns a JSON object where the key is the event counter, and the value +includes the number of children (and grandchildren) events. +Only events with children are included in the output. + +## Example + +e.g. Demo Job Template job +tuple(event counter, uuid, parent_uuid) + +``` +(1, '4598d19e-93b4-4e33-a0ae-b387a7348964', '') +(2, 'aae0d189-e3cb-102a-9f00-000000000006', '4598d19e-93b4-4e33-a0ae-b387a7348964') +(3, 'aae0d189-e3cb-102a-9f00-00000000000c', 'aae0d189-e3cb-102a-9f00-000000000006') +(4, 'f4194f14-e406-4124-8519-0fdb08b18f4b', 'aae0d189-e3cb-102a-9f00-00000000000c') +(5, '39f7ad99-dbf3-41e0-93f8-9999db4004f2', 'aae0d189-e3cb-102a-9f00-00000000000c') +(6, 'aae0d189-e3cb-102a-9f00-000000000008', 'aae0d189-e3cb-102a-9f00-000000000006') +(7, '39a49992-5ca4-4b6c-b178-e56d0b0333da', 'aae0d189-e3cb-102a-9f00-000000000008') +(8, '504f3b28-3ea8-4f6f-bd82-60cf8e807cc0', 'aae0d189-e3cb-102a-9f00-000000000008') +(9, 'a242be54-ebe6-4021-afab-f2878bff2e9f', '4598d19e-93b4-4e33-a0ae-b387a7348964') +``` + +output + +``` +{ +"1": { + "rowNumber": 0, + "numChildren": 8 +}, +"2": { + "rowNumber": 1, + "numChildren": 6 +}, +"3": { + "rowNumber": 2, + "numChildren": 2 +}, +"6": { + "rowNumber": 5, + "numChildren": 2 +} +} +"meta_event_nested_parent_uuid": {} +} +``` + +counter 1 is event 0, and has 8 children +counter 2 is event 1, and has 6 children +etc. + +The UI also needs to be able to collapse over "meta" events -- events that +show up due to verbosity or warnings from the system while the play is running. +These events have a 0 level event, with no parent uuid. + +``` +playbook_on_start +verbose + playbook_on_play_start + playbook_on_task_start + runner_on_start <- level 3 +verbose <- jump to level 0 +verbose + runner_on_ok <- jump back to level 3 + playbook_on_task_start + runner_on_start + runner_on_ok +verbose +verbose + playbook_on_stats +``` + +These verbose statements that fall in the middle of a series of children events +are problematic for the UI. +To help, this view will attempt to place the events into the hierarchy, without +the event level jumps. + +``` +playbook_on_start + verbose + playbook_on_play_start + playbook_on_task_start + runner_on_start <- A + verbose <- this maps to the uuid of A + verbose + runner_on_ok + playbook_on_task_start <- B + runner_on_start + runner_on_ok + verbose <- this maps to the uuid of B + verbose + playbook_on_stats +``` + +The output will include a JSON object where the key is the event counter, +and the value is the assigned nested uuid. diff --git a/awx/api/urls/job.py b/awx/api/urls/job.py index bea61a48a099..c6297600816b 100644 --- a/awx/api/urls/job.py +++ b/awx/api/urls/job.py @@ -10,6 +10,7 @@ JobRelaunch, JobCreateSchedule, JobJobHostSummariesList, + JobJobEventsChildrenSummary, JobJobEventsList, JobActivityStreamList, JobStdout, @@ -27,6 +28,7 @@ re_path(r'^(?P[0-9]+)/create_schedule/$', JobCreateSchedule.as_view(), name='job_create_schedule'), re_path(r'^(?P[0-9]+)/job_host_summaries/$', JobJobHostSummariesList.as_view(), name='job_job_host_summaries_list'), re_path(r'^(?P[0-9]+)/job_events/$', JobJobEventsList.as_view(), name='job_job_events_list'), + re_path(r'^(?P[0-9]+)/job_events/children_summary/$', JobJobEventsChildrenSummary.as_view(), name='job_job_events_children_summary'), re_path(r'^(?P[0-9]+)/activity_stream/$', JobActivityStreamList.as_view(), name='job_activity_stream_list'), re_path(r'^(?P[0-9]+)/stdout/$', JobStdout.as_view(), name='job_stdout'), re_path(r'^(?P[0-9]+)/notifications/$', JobNotificationsList.as_view(), name='job_notifications_list'), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index dda6a047962f..7c840d1f21da 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3842,6 +3842,84 @@ def get_queryset(self): return job.get_event_queryset().select_related('host').order_by('start_line') +class JobJobEventsChildrenSummary(APIView): + + renderer_classes = [JSONRenderer] + meta_events = ('debug', 'verbose', 'warning', 'error', 'system_warning', 'deprecated') + + def get(self, request, **kwargs): + resp = dict(children_summary={}, meta_event_nested_uuid={}, event_processing_finished=False) + job = get_object_or_404(models.Job, pk=kwargs['pk']) + if not job.event_processing_finished: + return Response(resp) + else: + resp["event_processing_finished"] = True + + events = list(job.get_event_queryset().values('counter', 'uuid', 'parent_uuid', 'event').order_by('counter')) + if len(events) == 0: + return Response(resp) + + # key is counter, value is number of total children (including children of children, etc.) + map_counter_children_tally = {i['counter']: {"rowNumber": 0, "numChildren": 0} for i in events} + # key is uuid, value is counter + map_uuid_counter = {i['uuid']: i['counter'] for i in events} + # key is uuid, value is parent uuid. Used as a quick lookup + map_uuid_puuid = {i['uuid']: i['parent_uuid'] for i in events} + # key is counter of meta events (i.e. verbose), value is uuid of the assigned parent + map_meta_counter_nested_uuid = {} + + prev_non_meta_event = events[0] + for i, e in enumerate(events): + if not e['event'] in JobJobEventsChildrenSummary.meta_events: + prev_non_meta_event = e + if not e['uuid']: + continue + puuid = e['parent_uuid'] + + # if event is verbose (or debug, etc), we need to "assign" it a + # parent. This code looks at the event level of the previous + # non-verbose event, and the level of the next (by looking ahead) + # non-verbose event. The verbose event is assigned the same parent + # uuid of the higher level event. + # e.g. + # E1 + # E2 + # verbose + # verbose <- we are on this event currently + # E4 + # We'll compare E2 and E4, and the verbose event + # will be assigned the parent uuid of E4 (higher event level) + if e['event'] in JobJobEventsChildrenSummary.meta_events: + event_level_before = models.JobEvent.LEVEL_FOR_EVENT[prev_non_meta_event['event']] + # find next non meta event + z = i + next_non_meta_event = events[-1] + while z < len(events): + if events[z]['event'] not in JobJobEventsChildrenSummary.meta_events: + next_non_meta_event = events[z] + break + z += 1 + event_level_after = models.JobEvent.LEVEL_FOR_EVENT[next_non_meta_event['event']] + if event_level_after and event_level_after > event_level_before: + puuid = next_non_meta_event['parent_uuid'] + else: + puuid = prev_non_meta_event['parent_uuid'] + if puuid: + map_meta_counter_nested_uuid[e['counter']] = puuid + map_counter_children_tally[e['counter']]['rowNumber'] = i + if not puuid: + continue + # now traverse up the parent, grandparent, etc. events and tally those + while puuid: + map_counter_children_tally[map_uuid_counter[puuid]]['numChildren'] += 1 + puuid = map_uuid_puuid.get(puuid, None) + + # create new dictionary, dropping events with 0 children + resp["children_summary"] = {k: v for k, v in map_counter_children_tally.items() if v['numChildren'] != 0} + resp["meta_event_nested_uuid"] = map_meta_counter_nested_uuid + return Response(resp) + + class AdHocCommandList(ListCreateAPIView): model = models.AdHocCommand diff --git a/awx/main/tests/functional/api/test_events.py b/awx/main/tests/functional/api/test_events.py index 43b31cb86b49..ce65a20d8030 100644 --- a/awx/main/tests/functional/api/test_events.py +++ b/awx/main/tests/functional/api/test_events.py @@ -46,3 +46,41 @@ def test_ad_hoc_events_sublist_truncation(get, organization_factory, job_templat response = get(url, user=objs.superusers.admin, expect=200) assert (len(response.data['results'][0]['stdout']) == 1025) == expected + + +@pytest.mark.django_db +def test_job_job_events_children_summary(get, organization_factory, job_template_factory): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, inventory='test_inv', project='test_proj').job_template + job = jt.create_unified_job() + url = reverse('api:job_job_events_children_summary', kwargs={'pk': job.pk}) + response = get(url, user=objs.superusers.admin, expect=200) + assert response.data["event_processing_finished"] == False + ''' + E1 + E2 + E3 + E4 (verbose) + E5 + ''' + JobEvent.create_from_data( + job_id=job.pk, uuid='uuid1', parent_uuid='', event="playbook_on_start", counter=1, stdout='a' * 1024, job_created=job.created + ).save() + JobEvent.create_from_data( + job_id=job.pk, uuid='uuid2', parent_uuid='uuid1', event="playbook_on_play_start", counter=2, stdout='a' * 1024, job_created=job.created + ).save() + JobEvent.create_from_data( + job_id=job.pk, uuid='uuid3', parent_uuid='uuid2', event="runner_on_start", counter=3, stdout='a' * 1024, job_created=job.created + ).save() + JobEvent.create_from_data(job_id=job.pk, uuid='uuid4', parent_uuid='', event='verbose', counter=4, stdout='a' * 1024, job_created=job.created).save() + JobEvent.create_from_data( + job_id=job.pk, uuid='uuid5', parent_uuid='uuid1', event="playbook_on_task_start", counter=5, stdout='a' * 1024, job_created=job.created + ).save() + job.emitted_events = job.get_event_queryset().count() + job.status = "successful" + job.save() + url = reverse('api:job_job_events_children_summary', kwargs={'pk': job.pk}) + response = get(url, user=objs.superusers.admin, expect=200) + assert response.data["children_summary"] == {1: {"rowNumber": 0, "numChildren": 4}, 2: {"rowNumber": 1, "numChildren": 2}} + assert response.data["meta_event_nested_uuid"] == {4: "uuid2"} + assert response.data["event_processing_finished"] == True