Skip to content

Commit

Permalink
[dagit] Add a Lineage tab to the Asset Details page (#8143)
Browse files Browse the repository at this point in the history 
* [dagit] Add last materialization, latest run columns to the asset table

* [dagit] Remove the Asset Grid prototype that was behind a feature flag (#8109)

Co-authored-by: bengotow <bgotow@elementl.com>

* [dagit] Remove the global asset graph in preparation for asset group graphs

* [dagit] Asset lineage

* Swap out the asset icon

* Lineage tab

* Asset lineage

* Loading states and polish

* Fix tests

* Add “show secondary edges” option, update icons

* Fix vertical divider lines on asset details page

* PR feedback

Co-authored-by: bengotow <bgotow@elementl.com>
  • Loading branch information
@bengotow
bengotow and bengotow committed Jun 3, 2022
1 parent 974af9f commit ccab3c0
Show file tree
Hide file tree
Showing 33 changed files with 567 additions and 2,433 deletions.
144 changes: 61 additions & 83 deletions js_modules/dagit/packages/core/src/asset-graph/AssetGraphExplorer.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {Box, Checkbox, Colors, NonIdealState, SplitPanelContainer} from '@dagster-io/ui';
import {Box, Checkbox, NonIdealState, SplitPanelContainer} from '@dagster-io/ui';
import flatMap from 'lodash/flatMap';
import pickBy from 'lodash/pickBy';
import uniq from 'lodash/uniq';
Expand All @@ -15,12 +15,11 @@ import {
QueryRefreshState,
useQueryRefreshAtInterval,
} from '../app/QueryRefresh';
import {withMiddleTruncation} from '../app/Util';
import {LaunchAssetExecutionButton} from '../assets/LaunchAssetExecutionButton';
import {AssetKey} from '../assets/types';
import {SVGViewport} from '../graph/SVGViewport';
import {useAssetLayout} from '../graph/asyncGraphLayout';
import {closestNodeInDirection, isNodeOffscreen} from '../graph/common';
import {closestNodeInDirection} from '../graph/common';
import {useDocumentTitle} from '../hooks/useDocumentTitle';
import {
GraphExplorerOptions,
Expand All @@ -43,7 +42,7 @@ import {GraphQueryInput} from '../ui/GraphQueryInput';
import {Loading} from '../ui/Loading';

import {AssetConnectedEdges} from './AssetEdges';
import {AssetNode, AssetNodeMinimal, NameMinimal} from './AssetNode';
import {AssetNode, AssetNodeMinimal} from './AssetNode';
import {ForeignNode} from './ForeignNode';
import {OmittedAssetsNotice} from './OmittedAssetsNotice';
import {SidebarAssetInfo} from './SidebarAssetInfo';
Expand All @@ -54,7 +53,6 @@ import {
GraphNode,
isSourceAsset,
tokenForAssetKey,
displayNameForAssetKey,
buildComputeStatusData,
} from './Utils';
import {AssetGraphLayout} from './layout';
Expand Down Expand Up @@ -86,7 +84,7 @@ export const AssetGraphExplorer: React.FC<Props> = (props) => {
graphAssetKeys,
allAssetKeys,
applyingEmptyDefault,
} = useAssetGraphData(props.pipelineSelector, props.explorerPath.opsQuery);
} = useAssetGraphData(props.explorerPath.opsQuery, {pipelineSelector: props.pipelineSelector});

const {liveResult, liveDataByNode} = useLiveDataForAssetKeys(
assetGraphData?.nodes,
Expand Down Expand Up @@ -132,7 +130,7 @@ export const AssetGraphExplorer: React.FC<Props> = (props) => {
);
};

const AssetGraphExplorerWithData: React.FC<
export const AssetGraphExplorerWithData: React.FC<
{
allAssetKeys: AssetKey[];
assetGraphData: GraphData;
Expand Down Expand Up @@ -169,6 +167,10 @@ const AssetGraphExplorerWithData: React.FC<
? selectedGraphNodes
: Object.values(assetGraphData.nodes).filter((a) => !isSourceAsset(a.definition));

const {layout, loading, async} = useAssetLayout(assetGraphData);

const viewportEl = React.useRef<SVGViewport>();

const onSelectNode = React.useCallback(
async (
e: React.MouseEvent<any> | React.KeyboardEvent<any>,
Expand Down Expand Up @@ -226,6 +228,11 @@ const AssetGraphExplorerWithData: React.FC<
).join(',');
}

const nextCenter = layout?.nodes[nextOpsNameSelection[nextOpsNameSelection.length - 1]];
if (nextCenter) {
viewportEl.current?.zoomToSVGCoords(nextCenter.bounds.x, nextCenter.bounds.y, true);
}

onChangeExplorerPath(
{
...explorerPath,
Expand All @@ -245,13 +252,10 @@ const AssetGraphExplorerWithData: React.FC<
history,
lastSelectedNode,
assetGraphData,
layout,
],
);

const {layout, loading, async} = useAssetLayout(assetGraphData);

const viewportEl = React.useRef<SVGViewport>();

const [lastRenderedLayout, setLastRenderedLayout] = React.useState<AssetGraphLayout | null>(null);
const renderingNewLayout = lastRenderedLayout !== layout;

Expand Down Expand Up @@ -328,23 +332,13 @@ const AssetGraphExplorerWithData: React.FC<
maxZoom={1.2}
maxAutocenterZoom={1.0}
>
{({scale: _scale}, viewportRect) => (
{({scale: _scale}) => (
<SVGContainer width={layout.width} height={layout.height}>
<AssetConnectedEdges highlighted={highlighted} edges={layout.edges} />

{Object.values(layout.nodes).map(({id, bounds}, index) => {
{Object.values(layout.nodes).map(({id, bounds}) => {
const graphNode = assetGraphData.nodes[id];
const path = JSON.parse(id);
if (!renderingNewLayout && isNodeOffscreen(bounds, viewportRect)) {
return id === lastSelectedNode?.id ? (
<RecenterGraph
key={index}
viewportRef={viewportEl}
x={bounds.x + bounds.width / 2}
y={bounds.y + bounds.height / 2}
/>
) : null;
}

return (
<foreignObject
Expand All @@ -363,16 +357,9 @@ const AssetGraphExplorerWithData: React.FC<
<ForeignNode assetKey={{path}} />
) : _scale < EXPERIMENTAL_MINI_SCALE ? (
<AssetNodeMinimal
style={{background: Colors.White}}
definition={graphNode.definition}
selected={selectedGraphNodes.includes(graphNode)}
>
<NameMinimal style={{fontSize: 28}}>
{withMiddleTruncation(
displayNameForAssetKey(graphNode.definition.assetKey),
{maxLength: 17},
)}
</NameMinimal>
</AssetNodeMinimal>
/>
) : (
<AssetNode
definition={graphNode.definition}
Expand Down Expand Up @@ -414,46 +401,52 @@ const AssetGraphExplorerWithData: React.FC<
</OptionsOverlay>
)}

<Box
flex={{direction: 'column', alignItems: 'flex-end', gap: 8}}
style={{position: 'absolute', right: 12, top: 12}}
>
<Box flex={{alignItems: 'center', gap: 12}}>
<QueryRefreshCountdown
refreshState={liveDataRefreshState}
dataDescription="materializations"
/>

<LaunchAssetExecutionButton
title={titleForLaunch(selectedGraphNodes, liveDataByNode)}
preferredJobName={explorerPath.pipelineName}
assets={launchGraphNodes.map((n) => n.definition)}
upstreamAssetKeys={uniqBy(
flatMap(launchGraphNodes.map((n) => n.definition.dependencyKeys)),
(key) => JSON.stringify(key),
).filter(
(key) =>
!launchGraphNodes.some(
(n) => JSON.stringify(n.assetKey) === JSON.stringify(key),
),
)}
/>
{setOptions && (
<Box
flex={{direction: 'column', alignItems: 'flex-end', gap: 8}}
style={{position: 'absolute', right: 12, top: 12}}
>
<Box flex={{alignItems: 'center', gap: 12}}>
<QueryRefreshCountdown
refreshState={liveDataRefreshState}
dataDescription="materializations"
/>

<LaunchAssetExecutionButton
title={titleForLaunch(selectedGraphNodes, liveDataByNode)}
preferredJobName={explorerPath.pipelineName}
assets={launchGraphNodes.map((n) => n.definition)}
upstreamAssetKeys={uniqBy(
flatMap(launchGraphNodes.map((n) => n.definition.dependencyKeys)),
(key) => JSON.stringify(key),
).filter(
(key) =>
!launchGraphNodes.some(
(n) => JSON.stringify(n.assetKey) === JSON.stringify(key),
),
)}
/>
</Box>
{!props.pipelineSelector && <OmittedAssetsNotice assetKeys={props.allAssetKeys} />}
</Box>
{!props.pipelineSelector && <OmittedAssetsNotice assetKeys={props.allAssetKeys} />}
</Box>
<QueryOverlay>
<GraphQueryInput
items={graphQueryItems}
value={explorerPath.opsQuery}
placeholder="Type an asset subset…"
onChange={(opsQuery) => onChangeExplorerPath({...explorerPath, opsQuery}, 'replace')}
popoverPosition="bottom-left"
/>
</QueryOverlay>
)}
{setOptions && (
<QueryOverlay>
<GraphQueryInput
items={graphQueryItems}
value={explorerPath.opsQuery}
placeholder="Type an asset subset…"
onChange={(opsQuery) =>
onChangeExplorerPath({...explorerPath, opsQuery}, 'replace')
}
popoverPosition="bottom-left"
/>
</QueryOverlay>
)}
</>
}
second={
selectedGraphNodes.length === 1 && selectedGraphNodes[0] ? (
!options.enableSidebar ? null : selectedGraphNodes.length === 1 && selectedGraphNodes[0] ? (
<RightInfoPanel>
<RightInfoPanelContent>
<SidebarAssetInfo
Expand Down Expand Up @@ -548,18 +541,3 @@ const titleForLaunch = (nodes: GraphNode[], liveDataByNode: LiveData) => {
nodes.length === 0 ? `All` : nodes.length === 1 ? `Selected` : `Selected (${nodes.length})`
}`;
};

// This is similar to react-router's "<Redirect />" in that it immediately performs
// the action you rendered.
//
const RecenterGraph: React.FC<{
viewportRef: React.MutableRefObject<SVGViewport | undefined>;
x: number;
y: number;
}> = ({viewportRef, x, y}) => {
React.useEffect(() => {
viewportRef.current?.zoomToSVGCoords(x, y, true);
}, [viewportRef, x, y]);

return <span />;
};
47 changes: 28 additions & 19 deletions js_modules/dagit/packages/core/src/asset-graph/AssetNode.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {gql} from '@apollo/client';
import {Colors, Icon, Tooltip, FontFamily, Box, CaptionMono, Spinner} from '@dagster-io/ui';
import isEqual from 'lodash/isEqual';
import React, {CSSProperties} from 'react';
import React from 'react';
import {Link} from 'react-router-dom';
import styled from 'styled-components/macro';

Expand Down Expand Up @@ -136,23 +136,16 @@ export const AssetNode: React.FC<{

export const AssetNodeMinimal: React.FC<{
selected: boolean;
style?: CSSProperties;
}> = ({selected, style, children}) => {
definition: AssetNodeFragment;
}> = ({selected, definition}) => {
return (
<AssetNodeContainer $selected={selected} style={{position: 'absolute', borderRadius: 12}}>
<AssetNodeBox
$selected={selected}
style={{
border: `4px solid ${Colors.Blue200}`,
borderRadius: 10,
position: 'absolute',
inset: 4,
...style,
}}
>
{children}
</AssetNodeBox>
</AssetNodeContainer>
<MinimalAssetNodeContainer $selected={selected}>
<MinimalAssetNodeBox $selected={selected}>
<MinimalName style={{fontSize: 28}}>
{withMiddleTruncation(displayNameForAssetKey(definition.assetKey), {maxLength: 17})}
</MinimalName>
</MinimalAssetNodeBox>
</MinimalAssetNodeContainer>
);
};

Expand Down Expand Up @@ -191,6 +184,7 @@ export const ASSET_NODE_FRAGMENT = gql`
id
...AssetNodeConfigFragment
graphName
jobNames
opNames
description
partitionDefinition
Expand Down Expand Up @@ -229,7 +223,7 @@ const AssetNodeContainer = styled.div<{$selected: boolean}>`
margin-bottom: 2px;
`;

export const AssetNodeBox = styled.div<{$selected: boolean}>`
const AssetNodeBox = styled.div<{$selected: boolean}>`
border: 2px solid ${(p) => (p.$selected ? Colors.Blue500 : Colors.Blue200)};
background: ${BoxColors.Stats};
border-radius: 5px;
Expand All @@ -251,7 +245,22 @@ const Name = styled.div`
gap: 4px;
`;

export const NameMinimal = styled(Name)`
const MinimalAssetNodeContainer = styled(AssetNodeContainer)`
position: absolute;
border-radius: 12px;
outline-offset: 2px;
outline-width: 4px;
`;

const MinimalAssetNodeBox = styled(AssetNodeBox)`
background: ${Colors.White};
border: 4px solid ${Colors.Blue200};
border-radius: 10px;
position: absolute;
inset: 4px;
`;

const MinimalName = styled(Name)`
font-weight: 600;
white-space: nowrap;
position: absolute;
Expand Down
45 changes: 1 addition & 44 deletions js_modules/dagit/packages/core/src/asset-graph/Utils.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import {pathVerticalDiagonal} from '@vx/shape';

import {AssetNodeDefinitionFragment} from '../assets/types/AssetNodeDefinitionFragment';

import {
AssetGraphLiveQuery_assetsLatestInfo,
AssetGraphLiveQuery_assetsLatestInfo_latestRun,
Expand Down Expand Up @@ -29,7 +27,7 @@ export function isHiddenAssetGroupJob(jobName: string) {
// because JSON.stringify's whitespace behavior is different than Python's.
//
export type GraphId = string;
export const toGraphId = (key: AssetKey): GraphId => JSON.stringify(key.path);
export const toGraphId = (key: {path: string[]}): GraphId => JSON.stringify(key.path);

export interface GraphNode {
id: GraphId;
Expand Down Expand Up @@ -83,47 +81,6 @@ export const buildGraphData = (assetNodes: AssetNode[]) => {
return data;
};

export const buildGraphDataFromSingleNode = (assetNode: AssetNodeDefinitionFragment) => {
const id = toGraphId(assetNode.assetKey);
const graphData: GraphData = {
downstream: {
[id]: {},
},
nodes: {
[id]: {
id,
assetKey: assetNode.assetKey,
definition: {...assetNode, dependencyKeys: [], dependedByKeys: []},
},
},
upstream: {
[id]: {},
},
};

for (const {asset} of assetNode.dependencies) {
const depId = toGraphId(asset.assetKey);
graphData.upstream[id][depId] = true;
graphData.downstream[depId] = {...graphData.downstream[depId], [id]: true};
graphData.nodes[depId] = {
id: depId,
assetKey: asset.assetKey,
definition: {...asset, dependencyKeys: [], dependedByKeys: []},
};
}
for (const {asset} of assetNode.dependedBy) {
const depId = toGraphId(asset.assetKey);
graphData.upstream[depId] = {...graphData.upstream[depId], [id]: true};
graphData.downstream[id][depId] = true;
graphData.nodes[depId] = {
id: depId,
assetKey: asset.assetKey,
definition: {...asset, dependencyKeys: [], dependedByKeys: []},
};
}
return graphData;
};

export const graphHasCycles = (graphData: GraphData) => {
const nodes = new Set(Object.keys(graphData.nodes));
const search = (stack: string[], node: string): boolean => {
Expand Down

1 comment on commit ccab3c0

@vercel
Copy link

@vercel vercel bot commented on ccab3c0 Jun 3, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

dagit-storybook – ./js_modules/dagit/packages/ui

dagit-storybook-git-master-elementl.vercel.app
dagit-storybook-elementl.vercel.app
dagit-storybook.vercel.app

Please sign in to comment.