Refactors the PartitionGraph component to be divorced from the actual…

… graphql query (#7616)

* add backfills by job, partitionRuns

* add tests for new graphql fields

* partition graph refactor

* fix test mock

* rename pipelineData => jobData

js_modules/dagit/packages/core/src/partitions/PartitionGraph.tsx

@@ -5,25 +5,23 @@ import styled from 'styled-components/macro';
 
 import {colorHash} from '../app/Util';
 
-import {PartitionGraphFragment} from './types/PartitionGraphFragment';
-
 type PointValue = number | null | undefined;
 type Point = {x: string; y: PointValue};
 
 interface PartitionGraphProps {
-  runsByPartitionName: {[name: string]: PartitionGraphFragment[]};
-  getPipelineDataForRun: (run: PartitionGraphFragment) => PointValue;
-  getStepDataForRun: (run: PartitionGraphFragment) => {[key: string]: PointValue[]};
+  partitionNames: string[];
+  jobDataByPartition?: {[partitionName: string]: PointValue};
+  stepDataByPartition?: {[partitionName: string]: {[key: string]: PointValue[]}};
   title?: string;
   yLabel?: string;
   isJob: boolean;
-  hiddenStepKeys: string[];
+  hiddenStepKeys?: string[];
 }
 
 export const PartitionGraph = ({
-  runsByPartitionName,
-  getPipelineDataForRun,
-  getStepDataForRun,
+  partitionNames,
+  jobDataByPartition,
+  stepDataByPartition,
   title,
   yLabel,
   isJob,
@@ -91,47 +89,38 @@ export const PartitionGraph = ({
         },
       },
       onClick: onGraphClick,
+      maintainAspectRatio: false,
     };
   }, [onGraphClick, title, yLabel]);
 
-  const selectRun = (runs?: PartitionGraphFragment[]) => {
-    if (!runs || !runs.length) {
-      return null;
-    }
-
-    // get most recent run
-    const toSort = runs.slice();
-    toSort.sort(_reverseSortRunCompare);
-    return toSort[0];
-  };
-
   const buildDatasetData = () => {
-    const pipelineData: Point[] = [];
+    const jobData: Point[] = [];
     const stepData = {};
 
-    const partitionNames = Object.keys(runsByPartitionName);
     partitionNames.forEach((partitionName) => {
-      const run = selectRun(runsByPartitionName[partitionName]);
       const hidden = !!hiddenPartitions[partitionName];
-      pipelineData.push({
-        x: partitionName,
-        y: run && !hidden ? getPipelineDataForRun(run) : undefined,
-      });
-
-      if (!run) {
-        return;
+      if (jobDataByPartition) {
+        jobData.push({
+          x: partitionName,
+          y: !hidden ? jobDataByPartition[partitionName] : undefined,
+        });
       }
 
-      const stepDataforRun = getStepDataForRun(run);
-      Object.keys(stepDataforRun).forEach((stepKey) => {
-        if (hiddenStepKeys.includes(stepKey)) {
-          return;
-        }
-        stepData[stepKey] = [
-          ...(stepData[stepKey] || []),
-          {x: partitionName, y: !hidden ? stepDataforRun[stepKey] : undefined},
-        ];
-      });
+      if (stepDataByPartition) {
+        const stepDataByKey = stepDataByPartition[partitionName];
+        Object.keys(stepDataByKey || {}).forEach((stepKey) => {
+          if (hiddenStepKeys?.includes(stepKey) || !stepDataByKey[stepKey]) {
+            return;
+          }
+          stepData[stepKey] = [
+            ...(stepData[stepKey] || []),
+            {
+              x: partitionName,
+              y: !hidden ? stepDataByKey[stepKey] : undefined,
+            },
+          ];
+        });
+      }
     });
 
     // stepData may have holes due to missing runs or missing steps.  For these to
@@ -140,20 +129,20 @@ export const PartitionGraph = ({
       stepData[stepKey] = _fillPartitions(partitionNames, stepData[stepKey]);
     });
 
-    return {pipelineData, stepData};
+    return {jobData, stepData};
   };
 
-  const {pipelineData, stepData} = buildDatasetData();
+  const {jobData, stepData} = buildDatasetData();
   const allLabel = isJob ? 'Total job' : 'Total pipeline';
   const graphData = {
-    labels: Object.keys(runsByPartitionName),
+    labels: partitionNames,
     datasets: [
-      ...(hiddenStepKeys.includes(allLabel)
+      ...(!jobDataByPartition || (hiddenStepKeys && hiddenStepKeys.includes(allLabel))
         ? []
         : [
             {
               label: allLabel,
-              data: pipelineData,
+              data: jobData,
               borderColor: Colors.Gray500,
               backgroundColor: 'rgba(0,0,0,0)',
             },
@@ -172,7 +161,7 @@ export const PartitionGraph = ({
   // We have a useMemo around the entire <PartitionGraphSet /> and there aren't many extra renders.
   return (
     <PartitionGraphContainer>
-      <Line type="line" data={() => graphData} height={100} options={defaultOptions} ref={chart} />
+      <Line type="line" data={() => graphData} height={300} options={defaultOptions} ref={chart} />
     </PartitionGraphContainer>
   );
 };
@@ -189,21 +178,9 @@ const _fillPartitions = (partitionNames: string[], points: Point[]) => {
   }));
 };
 
-const _reverseSortRunCompare = (a: PartitionGraphFragment, b: PartitionGraphFragment) => {
-  if (!a.stats || a.stats.__typename !== 'RunStatsSnapshot' || !a.stats.startTime) {
-    return 1;
-  }
-  if (!b.stats || b.stats.__typename !== 'RunStatsSnapshot' || !b.stats.startTime) {
-    return -1;
-  }
-  return b.stats.startTime - a.stats.startTime;
-};
-
 const PartitionGraphContainer = styled.div`
   display: flex;
   color: ${Colors.Gray700};
-  border-left: 1px solid ${Colors.KeylineGray};
-  border-bottom: 1px solid ${Colors.KeylineGray};
   padding: 24px 12px;
   text-decoration: none;
 `;

js_modules/dagit/packages/core/src/partitions/PartitionGraphSet.tsx

@@ -14,21 +14,60 @@ import {
   getStepExpectationRateForRun,
   getStepExpectationSuccessForRun,
   getStepMaterializationCountForRun,
-  PARTITION_GRAPH_FRAGMENT,
   StepSelector,
+  PARTITION_GRAPH_FRAGMENT,
 } from './PartitionGraphUtils';
 import {PartitionGraphSetRunFragment} from './types/PartitionGraphSetRunFragment';
 
+const _reverseSortRunCompare = (
+  a: PartitionGraphSetRunFragment,
+  b: PartitionGraphSetRunFragment,
+) => {
+  if (!a.stats || a.stats.__typename !== 'RunStatsSnapshot' || !a.stats.startTime) {
+    return 1;
+  }
+  if (!b.stats || b.stats.__typename !== 'RunStatsSnapshot' || !b.stats.startTime) {
+    return -1;
+  }
+  return b.stats.startTime - a.stats.startTime;
+};
+
 export const PartitionGraphSet: React.FC<{
   partitions: {name: string; runs: PartitionGraphSetRunFragment[]}[];
   isJob: boolean;
 }> = React.memo(({partitions, isJob}) => {
   const allStepKeys = getStepKeys(partitions);
   const [hiddenStepKeys, setHiddenStepKeys] = React.useState<string[]>([]);
 
-  const runsByPartitionName = {};
+  const partitionNames = partitions.map((x) => x.name);
+
+  const jobDurationData = {};
+  const stepDurationData = {};
+  const jobMaterializationData = {};
+  const stepMaterializationData = {};
+  const jobExpectationSuccessData = {};
+  const stepExpectationSuccessData = {};
+  const jobExpectationFailureData = {};
+  const stepExpectationFailureData = {};
+  const jobExpectationRateData = {};
+  const stepExpectationRateData = {};
+
   partitions.forEach((partition) => {
-    runsByPartitionName[partition.name] = partition.runs;
+    if (partition.runs && partition.runs.length) {
+      const toSort = partition.runs.slice();
+      toSort.sort(_reverseSortRunCompare);
+      const latestRun = toSort[0];
+      jobDurationData[partition.name] = getPipelineDurationForRun(latestRun);
+      stepDurationData[partition.name] = getStepDurationsForRun(latestRun);
+      jobMaterializationData[partition.name] = getPipelineMaterializationCountForRun(latestRun);
+      stepMaterializationData[partition.name] = getStepMaterializationCountForRun(latestRun);
+      jobExpectationSuccessData[partition.name] = getPipelineExpectationSuccessForRun(latestRun);
+      stepExpectationSuccessData[partition.name] = getStepExpectationSuccessForRun(latestRun);
+      jobExpectationFailureData[partition.name] = getPipelineExpectationFailureForRun(latestRun);
+      stepExpectationFailureData[partition.name] = getStepExpectationFailureForRun(latestRun);
+      jobExpectationRateData[partition.name] = getPipelineExpectationRateForRun(latestRun);
+      stepExpectationRateData[partition.name] = getStepExpectationRateForRun(latestRun);
+    }
   });
 
   return (
@@ -45,45 +84,46 @@ export const PartitionGraphSet: React.FC<{
           isJob={isJob}
           title="Execution Time by Partition"
           yLabel="Execution time (secs)"
-          runsByPartitionName={runsByPartitionName}
-          getPipelineDataForRun={getPipelineDurationForRun}
-          getStepDataForRun={getStepDurationsForRun}
+          partitionNames={partitionNames}
+          jobDataByPartition={jobDurationData}
+          stepDataByPartition={stepDurationData}
           hiddenStepKeys={hiddenStepKeys}
         />
+
         <PartitionGraph
           isJob={isJob}
           title="Materialization Count by Partition"
           yLabel="Number of materializations"
-          runsByPartitionName={runsByPartitionName}
-          getPipelineDataForRun={getPipelineMaterializationCountForRun}
-          getStepDataForRun={getStepMaterializationCountForRun}
+          partitionNames={partitionNames}
+          jobDataByPartition={jobMaterializationData}
+          stepDataByPartition={stepMaterializationData}
           hiddenStepKeys={hiddenStepKeys}
         />
         <PartitionGraph
           isJob={isJob}
           title="Expectation Successes by Partition"
           yLabel="Number of successes"
-          runsByPartitionName={runsByPartitionName}
-          getPipelineDataForRun={getPipelineExpectationSuccessForRun}
-          getStepDataForRun={getStepExpectationSuccessForRun}
+          partitionNames={partitionNames}
+          jobDataByPartition={jobExpectationSuccessData}
+          stepDataByPartition={stepExpectationSuccessData}
           hiddenStepKeys={hiddenStepKeys}
         />
         <PartitionGraph
           isJob={isJob}
           title="Expectation Failures by Partition"
           yLabel="Number of failures"
-          runsByPartitionName={runsByPartitionName}
-          getPipelineDataForRun={getPipelineExpectationFailureForRun}
-          getStepDataForRun={getStepExpectationFailureForRun}
+          partitionNames={partitionNames}
+          jobDataByPartition={jobExpectationFailureData}
+          stepDataByPartition={stepExpectationFailureData}
           hiddenStepKeys={hiddenStepKeys}
         />
         <PartitionGraph
           isJob={isJob}
           title="Expectation Rate by Partition"
           yLabel="Rate of success"
-          runsByPartitionName={runsByPartitionName}
-          getPipelineDataForRun={getPipelineExpectationRateForRun}
-          getStepDataForRun={getStepExpectationRateForRun}
+          partitionNames={partitionNames}
+          jobDataByPartition={jobExpectationRateData}
+          stepDataByPartition={stepExpectationRateData}
           hiddenStepKeys={hiddenStepKeys}
         />
       </div>
@@ -94,7 +134,6 @@ export const PartitionGraphSet: React.FC<{
 export const PARTITION_GRAPH_SET_RUN_FRAGMENT = gql`
   fragment PartitionGraphSetRunFragment on PipelineRun {
     id
-    status
     tags {
       key
       value

js_modules/dagit/packages/core/src/partitions/PartitionGraphUtils.tsx

@@ -8,7 +8,6 @@ import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorInfo';
 import {colorHash} from '../app/Util';
 
 import {PartitionGraphFragment} from './types/PartitionGraphFragment';
-
 export const PARTITION_GRAPH_FRAGMENT = gql`
   fragment PartitionGraphFragment on PipelineRun {
     id

js_modules/dagit/packages/core/src/partitions/PartitionRunMatrix.stories.tsx

@@ -5,7 +5,7 @@ import * as React from 'react';
 
 import {RunFilterToken} from '../runs/RunsFilterInput';
 import {StorybookProvider} from '../testing/StorybookProvider';
-import {RunStatus, StepEventStatus} from '../types/globalTypes';
+import {StepEventStatus} from '../types/globalTypes';
 
 import {PartitionRunMatrix} from './PartitionRunMatrix';
 import {
@@ -85,7 +85,6 @@ function buildRun(
     __typename: 'Run',
     id,
     runId: id,
-    status: RunStatus.SUCCESS,
     stats: {
       id,
       startTime,