Skip to content

Commit

Permalink
WIP: Don't fetch every single run in the InstanceBackfillsQuery (#7985)
Browse files Browse the repository at this point in the history 
Summary:
This elminates places where the (old) backfill ui loads every single run in every backfill. The problem doesn't appear to be the fact that we need to fetch all the runs, but rather the gigantic graphql response when we return 10s of thousands of runs to the client.

I did leave a 'cancelableRuns' field that I would love to remove as well - that appears to mainly exist so that Dagit can do buik termination from the client. Instead we should write a 'terminate every run in backfill X' bulk action and not need to send thousands of run IDs to dagit, but that is future work after this PR.

For now i just took the new backill UI out of the existing path to test this out. I won't land that - prha has graciously volunteered to split the two UIs a bit more before we land this.

I also still need to add tests for the new fields I added.

However, this drops a large backfill page with ~16k runs load time from about 4 minutes to about 15 seconds.
  • Loading branch information
@gibsondan
gibsondan committed May 20, 2022
1 parent d910d23 commit 4293e7c
Show file tree
Hide file tree
Showing 13 changed files with 542 additions and 178 deletions.
22 changes: 22 additions & 0 deletions js_modules/dagit/packages/core/src/graphql/schema.graphql
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 17 additions & 7 deletions js_modules/dagit/packages/core/src/instance/BackfillTable.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,13 +119,9 @@ export const BackfillTable = ({
}
};

const cancelableRuns = cancelRunBackfill?.runs.filter(
(run) => !doneStatuses.has(run?.status) && run.canTerminate,
);
const unfinishedRuns = cancelRunBackfill?.unfinishedRuns;
const unfinishedMap =
cancelRunBackfill?.runs
.filter((run) => !doneStatuses.has(run?.status))
.reduce((accum, run) => ({...accum, [run.id]: run.canTerminate}), {}) || {};
unfinishedRuns?.reduce((accum, run) => ({...accum, [run.id]: run.canTerminate}), {}) || {};

return (
<>
Expand Down Expand Up @@ -170,7 +166,7 @@ export const BackfillTable = ({
onComplete={() => refetch()}
/>
<TerminationDialog
isOpen={!!cancelableRuns?.length}
isOpen={!!unfinishedRuns?.length}
onClose={() => setCancelRunBackfill(undefined)}
onComplete={() => refetch()}
selectedRuns={unfinishedMap}
Expand Down Expand Up @@ -523,8 +519,18 @@ export const BACKFILL_TABLE_FRAGMENT = gql`
fragment BackfillTableFragment on PartitionBackfill {
backfillId
status
backfillStatus
numRequested
partitionNames
numPartitions
partitionRunStats {
numQueued
numInProgress
numSucceeded
numFailed
numPartitionsWithRuns
numTotalRuns
}
runs {
id
canTerminate
Expand All @@ -534,6 +540,10 @@ export const BACKFILL_TABLE_FRAGMENT = gql`
value
}
}
unfinishedRuns {
id
canTerminate
}
timestamp
partitionSetName
partitionSet {
Expand Down
15 changes: 7 additions & 8 deletions js_modules/dagit/packages/core/src/instance/BackfillTerminationDialog.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import {Button, DialogBody, DialogFooter, Dialog} from '@dagster-io/ui';
import * as React from 'react';

import {PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorInfo';
import {doneStatuses} from '../runs/RunStatuses';
import {TerminationDialog} from '../runs/TerminationDialog';
import {BulkActionStatus} from '../types/globalTypes';

Expand All @@ -26,13 +25,13 @@ export const BackfillTerminationDialog = ({backfill, onClose, onComplete}: Props
if (!backfill) {
return null;
}
const numUnscheduled = (backfill.partitionNames.length || 0) - (backfill.numRequested || 0);
const cancelableRuns = backfill.runs.filter(
(run) => !doneStatuses.has(run?.status) && run.canTerminate,
const numUnscheduled = (backfill.numPartitions || 0) - (backfill.numRequested || 0);
const unfinishedRuns = backfill.unfinishedRuns;

const unfinishedMap = unfinishedRuns?.reduce(
(accum, run) => ({...accum, [run.id]: run.canTerminate}),
{},
);
const unfinishedMap = backfill.runs
.filter((run) => !doneStatuses.has(run?.status))
.reduce((accum, run) => ({...accum, [run.id]: run.canTerminate}), {});

const cancel = async () => {
setIsSubmitting(true);
Expand Down Expand Up @@ -71,7 +70,7 @@ export const BackfillTerminationDialog = ({backfill, onClose, onComplete}: Props
isOpen={
!!backfill &&
(!numUnscheduled || backfill.status !== 'REQUESTED') &&
!!cancelableRuns.length
!!unfinishedRuns.length
}
onClose={onClose}
onComplete={onComplete}
Expand Down
98 changes: 34 additions & 64 deletions js_modules/dagit/packages/core/src/instance/InstanceBackfills.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,11 @@ import {PythonErrorInfo, PYTHON_ERROR_FRAGMENT} from '../app/PythonErrorInfo';
import {FIFTEEN_SECONDS, useQueryRefreshAtInterval} from '../app/QueryRefresh';
import {useDocumentTitle} from '../hooks/useDocumentTitle';
import {PipelineReference} from '../pipelines/PipelineReference';
import {
doneStatuses,
failedStatuses,
inProgressStatuses,
queuedStatuses,
successStatuses,
} from '../runs/RunStatuses';
import {DagsterTag} from '../runs/RunTag';
import {runsPathWithFilters} from '../runs/RunsFilterInput';
import {TerminationDialog} from '../runs/TerminationDialog';
import {useCursorPaginatedQuery} from '../runs/useCursorPaginatedQuery';
import {TimestampDisplay} from '../schedules/TimestampDisplay';
import {BulkActionStatus, RunStatus} from '../types/globalTypes';
import {BulkActionStatus} from '../types/globalTypes';
import {Loading} from '../ui/Loading';
import {isThisThingAJob, useRepository} from '../workspace/WorkspaceContext';
import {buildRepoAddress} from '../workspace/buildRepoAddress';
Expand All @@ -60,7 +52,6 @@ import {
InstanceBackfillsQueryVariables,
InstanceBackfillsQuery_partitionBackfillsOrError_PartitionBackfills_results,
InstanceBackfillsQuery_partitionBackfillsOrError_PartitionBackfills_results_partitionSet,
InstanceBackfillsQuery_partitionBackfillsOrError_PartitionBackfills_results_runs,
} from './types/InstanceBackfillsQuery';
import {
InstanceBackfillsQueryNew,
Expand All @@ -70,7 +61,6 @@ import {InstanceHealthForBackfillsQuery} from './types/InstanceHealthForBackfill
import {resumeBackfill, resumeBackfillVariables} from './types/resumeBackfill';

type Backfill = InstanceBackfillsQuery_partitionBackfillsOrError_PartitionBackfills_results;
type BackfillRun = InstanceBackfillsQuery_partitionBackfillsOrError_PartitionBackfills_results_runs;

const PAGE_SIZE = 25;

Expand Down Expand Up @@ -326,13 +316,9 @@ const BackfillTable = ({backfills, refetch}: {backfills: Backfill[]; refetch: ()
}
};

const cancelableRuns = cancelRunBackfill?.runs.filter(
(run) => !doneStatuses.has(run?.status) && run.canTerminate,
);
const unfinishedRuns = cancelRunBackfill?.unfinishedRuns;
const unfinishedMap =
cancelRunBackfill?.runs
.filter((run) => !doneStatuses.has(run?.status))
.reduce((accum, run) => ({...accum, [run.id]: run.canTerminate}), {}) || {};
unfinishedRuns?.reduce((accum, run) => ({...accum, [run.id]: run.canTerminate}), {}) || {};

return (
<>
Expand Down Expand Up @@ -364,7 +350,7 @@ const BackfillTable = ({backfills, refetch}: {backfills: Backfill[]; refetch: ()
onComplete={() => refetch()}
/>
<TerminationDialog
isOpen={!!cancelableRuns?.length}
isOpen={!!unfinishedRuns?.length}
onClose={() => setCancelRunBackfill(undefined)}
onComplete={() => refetch()}
selectedRuns={unfinishedMap}
Expand Down Expand Up @@ -418,7 +404,7 @@ const BackfillRow = ({
})
: null;

const canCancel = backfill.runs.some((run) => run.canTerminate);
const canCancel = backfill.unfinishedRuns.length > 0;

return (
<tr>
Expand Down Expand Up @@ -529,38 +515,16 @@ const TagButton = styled.button`
`;

const getProgressCounts = (backfill: Backfill) => {
const byPartitionRuns: {[key: string]: BackfillRun} = {};
backfill.runs.forEach((run) => {
const [runPartitionName] = run.tags
.filter((tag) => tag.key === DagsterTag.Partition)
.map((tag) => tag.value);

if (runPartitionName && !byPartitionRuns[runPartitionName]) {
byPartitionRuns[runPartitionName] = run;
}
});

const latestPartitionRuns = Object.values(byPartitionRuns);
const {numQueued, numInProgress, numSucceeded, numFailed} = latestPartitionRuns.reduce(
(accum: any, {status}: {status: RunStatus}) => {
return {
numQueued: accum.numQueued + (queuedStatuses.has(status) ? 1 : 0),
numInProgress: accum.numInProgress + (inProgressStatuses.has(status) ? 1 : 0),
numSucceeded: accum.numSucceeded + (successStatuses.has(status) ? 1 : 0),
numFailed: accum.numFailed + (failedStatuses.has(status) ? 1 : 0),
};
},
{numQueued: 0, numInProgress: 0, numSucceeded: 0, numFailed: 0},
);
const partitionRunStats = backfill.partitionRunStats;
const numTotal = backfill.numPartitions;

const numTotal = backfill.partitionNames.length;
return {
numQueued,
numInProgress,
numSucceeded,
numFailed,
numQueued: partitionRunStats.numQueued,
numInProgress: partitionRunStats.numInProgress,
numSucceeded: partitionRunStats.numSucceeded,
numFailed: partitionRunStats.numFailed,
numUnscheduled: numTotal - backfill.numRequested,
numSkipped: backfill.numRequested - latestPartitionRuns.length,
numSkipped: backfill.numRequested - partitionRunStats.numPartitionsWithRuns,
numTotal,
};
};
Expand Down Expand Up @@ -703,16 +667,20 @@ const BACKFILLS_QUERY = gql`
results {
backfillId
status
backfillStatus
numRequested
partitionNames
runs {
numPartitions
partitionRunStats {
numQueued
numInProgress
numSucceeded
numFailed
numPartitionsWithRuns
numTotalRuns
}
unfinishedRuns {
id
canTerminate
status
tags {
key
value
}
}
timestamp
partitionSetName
Expand All @@ -730,16 +698,13 @@ const BACKFILLS_QUERY = gql`
error {
...PythonErrorFragment
}
...BackfillTableFragment
}
}
...PythonErrorFragment
}
}
${PYTHON_ERROR_FRAGMENT}
${BACKFILL_TABLE_FRAGMENT}
`;

const BACKFILLS_QUERY_NEW = gql`
Expand All @@ -749,16 +714,21 @@ const BACKFILLS_QUERY_NEW = gql`
results {
backfillId
status
backfillStatus
numRequested
partitionNames
runs {
numPartitions
partitionRunStats {
numQueued
numInProgress
numSucceeded
numFailed
numPartitionsWithRuns
numTotalRuns
}
unfinishedRuns {
id
canTerminate
status
tags {
key
value
}
}
timestamp
partitionSetName
Expand Down
22 changes: 21 additions & 1 deletion js_modules/dagit/packages/core/src/instance/types/BackfillTableFragment.ts
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 4293e7c

Please sign in to comment.