[dagit] Async layout of large asset graphs (#7366)

js_modules/dagit/packages/core/jest/mocks/dagre_layout.worker.ts

@@ -1,12 +1,24 @@
 // eslint-disable-next-line no-restricted-imports
-import {layoutPipeline, IPipelineLayoutParams} from '../../src/graph/layout';
+import {ILayoutOp, layoutOpGraph} from '../../src/graph/layout';
+import {GraphData} from '../../src/workspace/asset-graph/Utils';
+import {layoutAssetGraph} from '../../src/workspace/asset-graph/layout';
+
+type WorkerMessageData =
+  | {type: 'layoutOpGraph'; ops: ILayoutOp[]; parentOp: ILayoutOp}
+  | {type: 'layoutAssetGraph'; graphData: GraphData};
 
 // eslint-disable-next-line import/no-default-export
 export default class MockWorker {
   onmessage = (_: any) => {};
 
   // mock expects data: { } instead of e: { data: { } }
-  postMessage(data: IPipelineLayoutParams) {
-    this.onmessage({data: layoutPipeline(data.solids)});
+  postMessage(data: WorkerMessageData) {
+    if (data.type === 'layoutOpGraph') {
+      const {ops, parentOp} = data;
+      this.onmessage({data: layoutOpGraph(ops, parentOp)});
+    } else if (data.type === 'layoutAssetGraph') {
+      const {graphData} = data;
+      this.onmessage({data: layoutAssetGraph(graphData)});
+    }
   }
 }

js_modules/dagit/packages/core/src/app/Util.tsx

@@ -74,14 +74,6 @@ export const formatElapsedTime = (msec: number) => {
   return `${hours}:${twoDigit(min)}:${twoDigit(sec)}`;
 };
 
-export function tokenForAssetKey(key: {path: string[]}) {
-  return key.path.join('>');
-}
-
-export function displayNameForAssetKey(key: {path: string[]}) {
-  return key.path.join(' > ');
-}
-
 export function breakOnUnderscores(str: string) {
   return str.replace(/_/g, '_\u200b');
 }

js_modules/dagit/packages/core/src/assets/AssetNodeDefinition.tsx

@@ -3,13 +3,18 @@ import {Box, Colors, Icon, Caption, Subheading, Mono} from '@dagster-io/ui';
 import * as React from 'react';
 import {Link} from 'react-router-dom';
 
-import {displayNameForAssetKey, tokenForAssetKey} from '../app/Util';
 import {DagsterTypeSummary} from '../dagstertype/DagsterType';
 import {Description} from '../pipelines/Description';
 import {instanceAssetsExplorerPathToURL} from '../pipelines/PipelinePathUtils';
 import {PipelineReference} from '../pipelines/PipelineReference';
 import {ASSET_NODE_FRAGMENT, ASSET_NODE_LIVE_FRAGMENT} from '../workspace/asset-graph/AssetNode';
-import {isSourceAsset, LiveData, __ASSET_GROUP} from '../workspace/asset-graph/Utils';
+import {
+  displayNameForAssetKey,
+  isSourceAsset,
+  LiveData,
+  tokenForAssetKey,
+  __ASSET_GROUP,
+} from '../workspace/asset-graph/Utils';
 import {buildRepoAddress} from '../workspace/buildRepoAddress';
 import {RepoAddress} from '../workspace/types';
 

js_modules/dagit/packages/core/src/assets/AssetTable.tsx

@@ -5,12 +5,12 @@ import {Link} from 'react-router-dom';
 import styled from 'styled-components/macro';
 
 import {usePermissions} from '../app/Permissions';
-import {tokenForAssetKey} from '../app/Util';
 import {useSelectionReducer} from '../hooks/useSelectionReducer';
 import {RepositoryLink} from '../nav/RepositoryLink';
 import {instanceAssetsExplorerPathToURL} from '../pipelines/PipelinePathUtils';
 import {MenuLink} from '../ui/MenuLink';
 import {markdownToPlaintext} from '../ui/markdownToPlaintext';
+import {tokenForAssetKey} from '../workspace/asset-graph/Utils';
 
 import {AssetLink} from './AssetLink';
 import {AssetWipeDialog} from './AssetWipeDialog';

js_modules/dagit/packages/core/src/assets/AssetView.tsx

@@ -17,7 +17,6 @@ import {
   QueryRefreshCountdown,
   useQueryRefreshAtInterval,
 } from '../app/QueryRefresh';
-import {displayNameForAssetKey} from '../app/Util';
 import {Timestamp} from '../app/time/Timestamp';
 import {useDocumentTitle} from '../hooks/useDocumentTitle';
 import {useQueryPersistedState} from '../hooks/useQueryPersistedState';
@@ -29,6 +28,7 @@ import {
   buildLiveData,
   REPOSITORY_LIVE_FRAGMENT,
   LiveData,
+  displayNameForAssetKey,
   toGraphId,
 } from '../workspace/asset-graph/Utils';
 import {buildRepoAddress} from '../workspace/buildRepoAddress';

js_modules/dagit/packages/core/src/assets/AssetWipeDialog.tsx

@@ -3,7 +3,7 @@ import {Button, DialogBody, DialogFooter, Dialog, Group} from '@dagster-io/ui';
 import * as React from 'react';
 
 import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorInfo';
-import {displayNameForAssetKey} from '../app/Util';
+import {displayNameForAssetKey} from '../workspace/asset-graph/Utils';
 
 interface AssetKey {
   path: string[];

js_modules/dagit/packages/core/src/assets/AssetsCatalogTable.tsx

@@ -10,12 +10,12 @@ import {
   QueryRefreshCountdown,
   useQueryRefreshAtInterval,
 } from '../app/QueryRefresh';
-import {tokenForAssetKey} from '../app/Util';
 import {useDocumentTitle} from '../hooks/useDocumentTitle';
 import {useQueryPersistedState} from '../hooks/useQueryPersistedState';
 import {RepoFilterButton} from '../instance/RepoFilterButton';
 import {Loading} from '../ui/Loading';
 import {DagsterRepoOption, WorkspaceContext} from '../workspace/WorkspaceContext';
+import {tokenForAssetKey} from '../workspace/asset-graph/Utils';
 import {buildRepoPath} from '../workspace/buildRepoAddress';
 
 import {AssetTable, ASSET_TABLE_FRAGMENT} from './AssetTable';

js_modules/dagit/packages/core/src/assets/PartitionHealthSummary.tsx

@@ -3,8 +3,8 @@ import {Tooltip, Spinner, Box, Colors} from '@dagster-io/ui';
 import {fromPairs} from 'lodash';
 import React from 'react';
 
-import {displayNameForAssetKey} from '../app/Util';
 import {assembleIntoSpans} from '../partitions/PartitionRangeInput';
+import {displayNameForAssetKey} from '../workspace/asset-graph/Utils';
 
 import {AssetKey} from './types';
 import {PartitionHealthQuery, PartitionHealthQueryVariables} from './types/PartitionHealthQuery';

js_modules/dagit/packages/core/src/graph/ExternalConnectionNode.tsx

@@ -5,7 +5,7 @@ import styled from 'styled-components/macro';
 
 import {Edge} from './OpLinks';
 import {SVGMonospaceText} from './SVGComponents';
-import {IPoint} from './getFullOpLayout';
+import {IPoint} from './asyncGraphLayout';
 import {isHighlighted} from './highlighting';
 
 interface ExternalConnectionNodeProps {

js_modules/dagit/packages/core/src/graph/MappingLine.tsx

@@ -1,7 +1,7 @@
 import * as React from 'react';
 
 import {Edge} from './OpLinks';
-import {IPoint} from './getFullOpLayout';
+import {IPoint} from './asyncGraphLayout';
 import {isHighlighted} from './highlighting';
 
 interface MappingLineProps {

js_modules/dagit/packages/core/src/graph/OpLinks.tsx

@@ -4,7 +4,7 @@ import styled from 'styled-components/macro';
 
 import {weakmapMemoize} from '../app/Util';
 
-import {IFullPipelineLayout, IFullOpLayout, ILayoutConnection} from './getFullOpLayout';
+import {OpGraphLayout, OpLayout, OpLayoutEdge} from './asyncGraphLayout';
 import {PipelineGraphOpFragment} from './types/PipelineGraphOpFragment';
 
 export type Edge = {a: string; b: string};
@@ -17,7 +17,7 @@ const buildSVGPath = pathVerticalDiagonal({
 });
 
 const buildSVGPaths = weakmapMemoize(
-  (connections: ILayoutConnection[], ops: {[name: string]: IFullOpLayout}) =>
+  (connections: OpLayoutEdge[], ops: {[name: string]: OpLayout}) =>
     connections.map(({from, to}) => {
       const sourceOutput = ops[from.opName].outputs[from.edgeName];
       if (!sourceOutput) {
@@ -67,8 +67,8 @@ export const OpLinks = React.memo(
   (props: {
     color: string;
     ops: PipelineGraphOpFragment[];
-    layout: IFullPipelineLayout;
-    connections: ILayoutConnection[];
+    layout: OpGraphLayout;
+    connections: OpLayoutEdge[];
     onHighlight: (arr: Edge[]) => void;
   }) => (
     <g>

js_modules/dagit/packages/core/src/graph/OpNode.tsx

@@ -3,18 +3,19 @@ import {Colors, Icon, FontFamily} from '@dagster-io/ui';
 import * as React from 'react';
 import styled from 'styled-components/macro';
 
-import {displayNameForAssetKey, withMiddleTruncation} from '../app/Util';
+import {withMiddleTruncation} from '../app/Util';
 import {AssetKey} from '../assets/types';
+import {displayNameForAssetKey} from '../workspace/asset-graph/Utils';
 
 import {OpIOBox, metadataForIO} from './OpIOBox';
 import {OpTags, IOpTag} from './OpTags';
-import {IFullOpLayout, ILayout} from './getFullOpLayout';
+import {OpLayout, ILayout} from './asyncGraphLayout';
 import {Edge} from './highlighting';
 import {OpNodeDefinitionFragment} from './types/OpNodeDefinitionFragment';
 import {OpNodeInvocationFragment} from './types/OpNodeInvocationFragment';
 
 interface IOpNodeProps {
-  layout: IFullOpLayout;
+  layout: OpLayout;
   invocation?: OpNodeInvocationFragment;
   definition: OpNodeDefinitionFragment;
   highlightedEdges: Edge[];

js_modules/dagit/packages/core/src/graph/ParentOpNode.tsx

@@ -10,12 +10,12 @@ import {MappingLine} from './MappingLine';
 import {metadataForCompositeParentIO, PARENT_IN, PARENT_OUT, OpIOBox} from './OpIOBox';
 import {position} from './OpNode';
 import {SVGLabeledRect} from './SVGComponents';
-import {IFullPipelineLayout} from './getFullOpLayout';
+import {OpGraphLayout} from './asyncGraphLayout';
 import {Edge} from './highlighting';
 import {PipelineGraphOpFragment} from './types/PipelineGraphOpFragment';
 
 interface ParentOpNodeProps {
-  layout: IFullPipelineLayout;
+  layout: OpGraphLayout;
   op: PipelineGraphOpFragment;
   minified: boolean;
 

js_modules/dagit/packages/core/src/graph/PipelineGraph.stories.tsx

@@ -5,7 +5,7 @@ import {OpNameOrPath} from '../ops/OpNameOrPath';
 
 import {PipelineGraph} from './PipelineGraph';
 import {SVGViewport} from './SVGViewport';
-import {getDagrePipelineLayout} from './getFullOpLayout';
+import {getFullOpLayout} from './asyncGraphLayout';
 import {PipelineGraphOpFragment} from './types/PipelineGraphOpFragment';
 
 // eslint-disable-next-line import/no-default-export
@@ -106,7 +106,7 @@ export const Basic = () => {
     <PipelineGraph
       pipelineName="Test Pipeline"
       ops={ops}
-      layout={getDagrePipelineLayout(ops)}
+      layout={getFullOpLayout(ops)}
       interactor={SVGViewport.Interactors.PanAndZoom}
       focusOps={ops.filter((s) => focusOps.includes(s.name))}
       highlightedOps={[]}
@@ -134,7 +134,7 @@ export const FanOut = () => {
     <PipelineGraph
       pipelineName="Test Pipeline"
       ops={ops}
-      layout={getDagrePipelineLayout(ops)}
+      layout={getFullOpLayout(ops)}
       interactor={SVGViewport.Interactors.PanAndZoom}
       focusOps={ops.filter((s) => focusOps.includes(s.name))}
       highlightedOps={[]}
@@ -159,7 +159,7 @@ export const Tagged = () => {
     <PipelineGraph
       pipelineName="Test Pipeline"
       ops={ops}
-      layout={getDagrePipelineLayout(ops)}
+      layout={getFullOpLayout(ops)}
       interactor={SVGViewport.Interactors.PanAndZoom}
       focusOps={ops.filter((s) => focusOps.includes(s.name))}
       highlightedOps={[]}
@@ -217,7 +217,7 @@ export const Composite = () => {
       ops={parentOp ? childOps : ops}
       parentOp={parentOp}
       parentHandleID={parentOpName}
-      layout={parentOp ? getDagrePipelineLayout(childOps, parentOp) : getDagrePipelineLayout(ops)}
+      layout={parentOp ? getFullOpLayout(childOps, parentOp) : getFullOpLayout(ops)}
       interactor={SVGViewport.Interactors.PanAndZoom}
       focusOps={ops.filter((s) => focusOps.includes(s.name))}
       highlightedOps={[]}

js_modules/dagit/packages/core/src/graph/PipelineGraph.tsx

@@ -9,15 +9,15 @@ import {OpLinks} from './OpLinks';
 import {OpNode, OP_NODE_DEFINITION_FRAGMENT, OP_NODE_INVOCATION_FRAGMENT} from './OpNode';
 import {ParentOpNode, SVGLabeledParentRect} from './ParentOpNode';
 import {DETAIL_ZOOM, SVGViewport, SVGViewportInteractor} from './SVGViewport';
-import {IFullPipelineLayout, IFullOpLayout, ILayout} from './getFullOpLayout';
+import {OpGraphLayout, OpLayout, ILayout} from './asyncGraphLayout';
 import {Edge, isHighlighted, isOpHighlighted} from './highlighting';
 import {PipelineGraphOpFragment} from './types/PipelineGraphOpFragment';
 
 const NoOp = () => {};
 
 interface IPipelineGraphProps {
   pipelineName: string;
-  layout: IFullPipelineLayout;
+  layout: OpGraphLayout;
   ops: PipelineGraphOpFragment[];
   focusOps: PipelineGraphOpFragment[];
   parentHandleID?: string;
@@ -35,7 +35,7 @@ interface IPipelineGraphProps {
 
 interface IPipelineContentsProps extends IPipelineGraphProps {
   minified: boolean;
-  layout: IFullPipelineLayout;
+  layout: OpGraphLayout;
   bounds: {top: number; left: number; right: number; bottom: number};
 }
 
@@ -44,7 +44,7 @@ interface IPipelineContentsProps extends IPipelineGraphProps {
  * an array of bounding boxes and common prefixes. Used to render lightweight
  * outlines around flattened composites.
  */
-function computeOpPrefixBoundingBoxes(layout: IFullPipelineLayout) {
+function computeOpPrefixBoundingBoxes(layout: OpGraphLayout) {
   const groups: {[base: string]: ILayout[]} = {};
   let maxDepth = 0;
 
@@ -109,20 +109,6 @@ const PipelineGraphContents: React.FC<IPipelineContentsProps> = React.memo((prop
           minified={minified}
         />
       )}
-      {/* {selectedOp && layout.ops[selectedOp.name] && (
-        // this rect is hidden beneath the user's selection with a React key so that
-        // when they expand the composite op React sees this component becoming
-        // the one above and re-uses the DOM node. This allows us to animate the rect's
-        // bounds from the parent layout to the inner layout with no React state.
-        <SVGLabeledParentRect
-          {...layout.ops[selectedOp.name].op}
-          key={`composite-rect-${selectedHandleID}`}
-          label={''}
-          fill={Colors.Gray50}
-          minified={true}
-        />
-      )} */}
-
       {parentOp && (
         <ParentOpNode
           onClickOp={onClickOp}
@@ -209,7 +195,7 @@ export class PipelineGraph extends React.Component<IPipelineGraphProps> {
 
   resolveOpPosition = (
     arg: OpNameOrPath,
-    cb: (cx: number, cy: number, layout: IFullOpLayout) => void,
+    cb: (cx: number, cy: number, layout: OpLayout) => void,
   ) => {
     const lastName = 'name' in arg ? arg.name : arg.path[arg.path.length - 1];
     const opLayout = this.props.layout.ops[lastName];
@@ -241,15 +227,15 @@ export class PipelineGraph extends React.Component<IPipelineGraphProps> {
     }
 
     const current = layout.ops[selectedOp.name];
-    const center = (op: IFullOpLayout): {x: number; y: number} => ({
+    const center = (op: OpLayout): {x: number; y: number} => ({
       x: op.boundingBox.x + op.boundingBox.width / 2,
       y: op.boundingBox.y + op.boundingBox.height / 2,
     });
 
     /* Sort all the ops in the graph based on their attractiveness
     as a jump target. We want the nearest node in the exact same row for left/right,
     and the visually "closest" node above/below for up/down. */
-    const score = (op: IFullOpLayout): number => {
+    const score = (op: OpLayout): number => {
       const dx = center(op).x - center(current).x;
       const dy = center(op).y - center(current).y;