feat(ingestion) Fetch live logs on an ingestion run from UI

datahub-project · Aug 16, 2022 · b78da41 · b78da41
1 parent 0d5e698
commit b78da41
Show file tree

Hide file tree

Showing 3 changed files with 73 additions and 22 deletions.
diff --git a/datahub-web-react/src/app/ingest/source/ExecutionRequestDetailsModal.tsx b/datahub-web-react/src/app/ingest/source/ExecutionRequestDetailsModal.tsx
@@ -11,6 +11,8 @@ import {
     getExecutionRequestStatusDisplayColor,
     getExecutionRequestStatusDisplayText,
     getExecutionRequestStatusIcon,
+    getExecutionRequestSummaryText,
+    RUNNING,
     SUCCESS,
 } from './utils';
 
@@ -89,7 +91,7 @@ type Props = {
 
 export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
     const [showExpandedLogs, setShowExpandedLogs] = useState(true);
-    const { data, loading, error } = useGetIngestionExecutionRequestQuery({ variables: { urn } });
+    const { data, loading, error, refetch } = useGetIngestionExecutionRequestQuery({ variables: { urn } });
     const output = data?.executionRequest?.result?.report || 'No output found.';
 
     useEffect(() => {
@@ -105,6 +107,14 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
     const logs = (showExpandedLogs && output) || output.slice(0, 100);
     const result = data?.executionRequest?.result?.status;
 
+    useEffect(() => {
+        const interval = setInterval(() => {
+            if (result === RUNNING) refetch();
+        }, 2000);
+
+        return () => clearInterval(interval);
+    });
+
     const ResultIcon = result && getExecutionRequestStatusIcon(result);
     const resultColor = result && getExecutionRequestStatusDisplayColor(result);
     const resultText = result && (
@@ -114,12 +124,9 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
         </Typography.Text>
     );
     const resultSummaryText =
-        (result && (
-            <Typography.Text type="secondary">
-                Ingestion {result === SUCCESS ? 'successfully completed' : 'completed with errors'}.
-            </Typography.Text>
-        )) ||
+        (result && <Typography.Text type="secondary">{getExecutionRequestSummaryText(result)}</Typography.Text>) ||
         undefined;
+    const isOutputExpandable = output.length > 100;
 
     return (
         <Modal
@@ -160,11 +167,20 @@ export const ExecutionDetailsModal = ({ urn, visible, onClose }: Props) => {
                         </Button>
                     </SectionSubHeader>
                     <Typography.Paragraph ellipsis>
-                        <pre>{`${logs}${!showExpandedLogs && '...'}`}</pre>
-                        {!showExpandedLogs && (
-                            <ShowMoreButton type="link" onClick={() => setShowExpandedLogs(true)}>
-                                Show More
-                            </ShowMoreButton>
+                        <pre>{`${logs}${!showExpandedLogs ? '...' : ''}`}</pre>
+                        {isOutputExpandable && (
+                            <>
+                                {!showExpandedLogs && (
+                                    <ShowMoreButton type="link" onClick={() => setShowExpandedLogs(true)}>
+                                        Show More
+                                    </ShowMoreButton>
+                                )}
+                                {showExpandedLogs && (
+                                    <ShowMoreButton type="link" onClick={() => setShowExpandedLogs(false)}>
+                                        Hide
+                                    </ShowMoreButton>
+                                )}
+                            </>
                         )}
                     </Typography.Paragraph>
                 </LogsSection>

diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -1,5 +1,5 @@
 import { CopyOutlined, DeleteOutlined, PlusOutlined, RedoOutlined } from '@ant-design/icons';
-import React, { useEffect, useState } from 'react';
+import React, { useCallback, useEffect, useState } from 'react';
 import * as QueryString from 'query-string';
 import { useLocation } from 'react-router';
 import { Button, Empty, Image, message, Modal, Pagination, Tooltip, Typography } from 'antd';
@@ -21,6 +21,7 @@ import {
     getExecutionRequestStatusDisplayColor,
     getExecutionRequestStatusDisplayText,
     getExecutionRequestStatusIcon,
+    RUNNING,
     sourceTypeToIconUrl,
 } from './utils';
 import { DEFAULT_EXECUTOR_ID, SourceBuilderState } from './builder/types';
@@ -118,11 +119,27 @@ export const IngestionSourceList = () => {
     const focusSource =
         (focusSourceUrn && filteredSources.find((source) => source.urn === focusSourceUrn)) || undefined;
 
-    const onRefresh = () => {
+    const onRefresh = useCallback(() => {
         refetch();
         // Used to force a re-render of the child execution request list.
         setLastRefresh(new Date().getMilliseconds());
-    };
+    }, [refetch]);
+
+    const [refreshInterval, setRefreshInterval] = useState<NodeJS.Timeout | null>(null);
+
+    useEffect(() => {
+        const runningSource = filteredSources.find((source) =>
+            source.executions?.executionRequests.find((request) => request.result?.status === RUNNING),
+        );
+        if (runningSource) {
+            if (!refreshInterval) {
+                const interval = setInterval(onRefresh, 3000);
+                setRefreshInterval(interval);
+            }
+        } else if (refreshInterval) {
+            clearInterval(refreshInterval);
+        }
+    }, [filteredSources, refreshInterval, onRefresh]);
 
     const executeIngestionSource = (urn: string) => {
         createExecutionRequestMutation({
@@ -137,7 +154,7 @@ export const IngestionSourceList = () => {
                     content: `Successfully submitted ingestion execution request!`,
                     duration: 3,
                 });
-                setInterval(() => onRefresh(), 3000);
+                setTimeout(() => onRefresh(), 3000);
             })
             .catch((e) => {
                 message.destroy();
@@ -391,13 +408,16 @@ export const IngestionSourceList = () => {
                     <Button style={{ marginRight: 16 }} onClick={() => onEdit(record.urn)}>
                         EDIT
                     </Button>
-                    <Button
-                        disabled={record.lastExecStatus === 'RUNNING'}
-                        style={{ marginRight: 16 }}
-                        onClick={() => onExecute(record.urn)}
-                    >
-                        EXECUTE
-                    </Button>
+                    {record.lastExecStatus !== RUNNING && (
+                        <Button style={{ marginRight: 16 }} onClick={() => onExecute(record.urn)}>
+                            EXECUTE
+                        </Button>
+                    )}
+                    {record.lastExecStatus === RUNNING && (
+                        <Button style={{ marginRight: 16 }} onClick={() => setFocusExecutionUrn(record.lastExecUrn)}>
+                            DETAILS
+                        </Button>
+                    )}
 
                     <Button onClick={() => onDelete(record.urn)} type="text" shape="circle" danger>
                         <DeleteOutlined />

diff --git a/datahub-web-react/src/app/ingest/source/utils.ts b/datahub-web-react/src/app/ingest/source/utils.ts
@@ -55,6 +55,21 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
     );
 };
 
+export const getExecutionRequestSummaryText = (status: string) => {
+    switch (status) {
+        case RUNNING:
+            return 'Ingestion is running';
+        case SUCCESS:
+            return 'Ingestion successfully completed';
+        case FAILURE:
+            return 'Ingestion completed with errors';
+        case CANCELLED:
+            return 'Ingestion was cancelled';
+        default:
+            return 'Ingestion status not recognized';
+    }
+};
+
 export const getExecutionRequestStatusDisplayColor = (status: string) => {
     return (
         (status === RUNNING && REDESIGN_COLORS.BLUE) ||