Skip to content

Commit

Permalink
[dagit] Show counts next to log filter tags (#8141)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hellendag
hellendag committed Jun 1, 2022
1 parent 8fc3838 commit 1d216c5
Show file tree
Hide file tree
Showing 9 changed files with 164 additions and 88 deletions.
2 changes: 1 addition & 1 deletion js_modules/dagit/packages/app/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"@types/node": "^16.11.20",
"@types/react": "17.0.4",
"@types/react-dom": "^17.0.11",
"eslint": "8.7.0",
"eslint": "8.6.0",
"eslint-plugin-jest": "^26.4.6",
"eslint-webpack-plugin": "3.1.1",
"prettier": "2.2.1",
Expand Down
94 changes: 54 additions & 40 deletions js_modules/dagit/packages/core/src/runs/LogsProvider.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ import * as React from 'react';
import {WebSocketContext} from '../app/WebSocketProvider';
import {RunStatus} from '../types/globalTypes';

import {LogLevelCounts} from './LogsToolbar';
import {RunFragments} from './RunFragments';
import {logNodeLevel} from './logNodeLevel';
import {LogNode} from './types';
import {
PipelineRunLogsSubscription,
PipelineRunLogsSubscriptionVariables,
Expand All @@ -27,11 +30,9 @@ export interface LogFilter {
hideNonMatches: boolean;
}

type LogNode = RunDagsterRunEventFragment & {clientsideKey: string};
type Nodes = LogNode[];

export interface LogsProviderLogs {
allNodes: LogNode[];
counts: LogLevelCounts;
loading: boolean;
}

Expand Down Expand Up @@ -62,32 +63,53 @@ const pipelineStatusFromMessages = (messages: RunDagsterRunEventFragment[]) => {
const BATCH_INTERVAL = 100;

type State = {
nodes: Nodes;
nodes: LogNode[];
cursor: string | null;
counts: LogLevelCounts;
loading: boolean;
};

type Action =
| {type: 'append'; queued: RunDagsterRunEventFragment[]; hasMore: boolean; cursor: string}
| {type: 'set-cursor'; cursor: string}
| {type: 'reset'};

const emptyCounts = {
DEBUG: 0,
INFO: 0,
WARNING: 0,
ERROR: 0,
CRITICAL: 0,
EVENT: 0,
};

const reducer = (state: State, action: Action) => {
switch (action.type) {
case 'append':
const nodes = [...state.nodes, ...action.queued].map((m, idx) => ({
...m,
clientsideKey: `csk${idx}`,
case 'append': {
const queuedNodes = action.queued.map((node, ii) => ({
...node,
clientsideKey: `csk${node.timestamp}-${ii}`,
}));
return {...state, nodes, loading: action.hasMore, cursor: action.cursor};
const nodes = [...state.nodes, ...queuedNodes];
const counts = {...state.counts};
queuedNodes.forEach((node) => {
const level = logNodeLevel(node);
counts[level]++;
});
return {nodes, counts, loading: action.hasMore, cursor: action.cursor};
}
case 'set-cursor':
return {...state, cursor: action.cursor};
case 'reset':
return {nodes: [], cursor: null, loading: true};
return {nodes: [], counts: emptyCounts, cursor: null, loading: true};
default:
return state;
}
};

const initialState = {
const initialState: State = {
nodes: [],
counts: emptyCounts,
cursor: null,
loading: true,
};
Expand Down Expand Up @@ -143,7 +165,7 @@ const useLogsProviderWithSubscription = (runId: string) => {
}, BATCH_INTERVAL);
}, []);

const {nodes, cursor, loading} = state;
const {nodes, counts, cursor, loading} = state;

useSubscription<PipelineRunLogsSubscription, PipelineRunLogsSubscriptionVariables>(
PIPELINE_RUN_LOGS_SUBSCRIPTION,
Expand Down Expand Up @@ -173,8 +195,8 @@ const useLogsProviderWithSubscription = (runId: string) => {
);

return React.useMemo(
() => (nodes !== null ? {allNodes: nodes, loading} : {allNodes: [], loading}),
[loading, nodes],
() => (nodes !== null ? {allNodes: nodes, counts, loading} : {allNodes: [], counts, loading}),
[counts, loading, nodes],
);
};

Expand All @@ -197,8 +219,8 @@ const POLL_INTERVAL = 5000;

const LogsProviderWithQuery = (props: LogsProviderWithQueryProps) => {
const {children, runId} = props;
const [nodes, setNodes] = React.useState<LogNode[]>(() => []);
const [cursor, setCursor] = React.useState<string | null>(null);
const [state, dispatch] = React.useReducer(reducer, initialState);
const {counts, cursor, nodes} = state;

const {stopPolling, startPolling} = useQuery<RunLogsQuery, RunLogsQueryVariables>(
RUN_LOGS_QUERY,
Expand All @@ -210,32 +232,24 @@ const LogsProviderWithQuery = (props: LogsProviderWithQueryProps) => {
// We have to stop polling in order to update the `after` value.
stopPolling();

const slice = () => {
const count = nodes.length;
if (data?.pipelineRunOrError.__typename === 'Run') {
return data?.pipelineRunOrError.eventConnection.events.map((event, ii) => ({
...event,
clientsideKey: `csk${count + ii}`,
}));
}
return [];
};

const newSlice = slice();
setNodes((current) => [...current, ...newSlice]);
if (data?.pipelineRunOrError.__typename === 'Run') {
setCursor(data.pipelineRunOrError.eventConnection.cursor);
if (data?.pipelineRunOrError.__typename !== 'Run') {
return;
}

const status =
data?.pipelineRunOrError.__typename === 'Run' ? data?.pipelineRunOrError.status : null;
const run = data.pipelineRunOrError;
const queued = run.eventConnection.events;
const status = run.status;
const cursor = run.eventConnection.cursor;

if (
status &&
const hasMore =
!!status &&
status !== RunStatus.FAILURE &&
status !== RunStatus.SUCCESS &&
status !== RunStatus.CANCELED
) {
status !== RunStatus.CANCELED;

dispatch({type: 'append', queued, hasMore, cursor});

if (hasMore) {
startPolling(POLL_INTERVAL);
}
},
Expand All @@ -246,8 +260,8 @@ const LogsProviderWithQuery = (props: LogsProviderWithQueryProps) => {
<>
{children(
nodes !== null && nodes.length > 0
? {allNodes: nodes, loading: false}
: {allNodes: [], loading: true},
? {allNodes: nodes, counts, loading: false}
: {allNodes: [], counts, loading: true},
)}
</>
);
Expand All @@ -263,7 +277,7 @@ export const LogsProvider: React.FC<LogsProviderProps> = (props) => {
}

if (availability === 'attempting-to-connect') {
return <>{children({allNodes: [], loading: true})}</>;
return <>{children({allNodes: [], counts: emptyCounts, loading: true})}</>;
}

return <LogsProviderWithSubscription runId={runId}>{children}</LogsProviderWithSubscription>;
Expand Down
3 changes: 2 additions & 1 deletion js_modules/dagit/packages/core/src/runs/LogsScrollingTable.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
import {ColumnWidthsProvider, Headers} from './LogsScrollingTableHeader';
import {IRunMetadataDict} from './RunMetadataProvider';
import {eventTypeToDisplayType} from './getRunFilterProviders';
import {logNodeLevel} from './logNodeLevel';
import {RunDagsterRunEventFragment} from './types/RunDagsterRunEventFragment';

const LOGS_PADDING_BOTTOM = 50;
Expand Down Expand Up @@ -52,7 +53,7 @@ function filterLogs(logs: LogsProviderLogs, filter: LogFilter, filterStepKeys: s
if (node.__typename === 'AssetMaterializationPlannedEvent') {
return false;
}
const l = node.__typename === 'LogMessageEvent' ? node.level : 'EVENT';
const l = logNodeLevel(node);
if (!filter.levels[l]) {
return false;
}
Expand Down
101 changes: 71 additions & 30 deletions js_modules/dagit/packages/core/src/runs/LogsToolbar.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,17 @@ import {
Spinner,
Tab,
Tabs,
Tag,
IconWrapper,
Colors,
Tooltip,
FontFamily,
} from '@dagster-io/ui';
import * as React from 'react';
import styled from 'styled-components/macro';

import {useCopyToClipboard} from '../app/browser';
import {OptionsContainer, OptionsDivider} from '../gantt/VizComponents';
import {compactNumber} from '../ui/formatters';

import {ExecutionStateDot} from './ExecutionStateDot';
import {LogLevel} from './LogLevel';
Expand All @@ -43,6 +44,7 @@ export enum LogType {
interface ILogsToolbarProps {
steps: string[];
metadata: IRunMetadataDict;
counts: LogLevelCounts;

filter: LogFilter;
onSetFilter: (filter: LogFilter) => void;
Expand All @@ -60,6 +62,7 @@ export const LogsToolbar: React.FC<ILogsToolbarProps> = (props) => {
const {
steps,
metadata,
counts,
filter,
onSetFilter,
logType,
Expand All @@ -86,7 +89,12 @@ export const LogsToolbar: React.FC<ILogsToolbarProps> = (props) => {
/>
<OptionsDivider />
{logType === 'structured' ? (
<StructuredLogToolbar filter={filter} onSetFilter={onSetFilter} steps={steps} />
<StructuredLogToolbar
counts={counts}
filter={filter}
onSetFilter={onSetFilter}
steps={steps}
/>
) : (
<ComputeLogToolbar
steps={steps}
Expand Down Expand Up @@ -239,12 +247,16 @@ const DownloadLink = styled.a`
}
`;

export type LogLevelCounts = Record<LogLevel, number>;

const StructuredLogToolbar = ({
filter,
counts,
onSetFilter,
steps,
}: {
filter: LogFilter;
counts: LogLevelCounts;
onSetFilter: (filter: LogFilter) => void;
steps: string[];
}) => {
Expand Down Expand Up @@ -305,35 +317,32 @@ const StructuredLogToolbar = ({
/>
) : null}
<OptionsDivider />
<Group direction="row" spacing={4} alignItems="center">
<Box flex={{direction: 'row', alignItems: 'center', gap: 8}}>
{Object.keys(LogLevel).map((level) => {
const enabled = filter.levels[level];
return (
<FilterButton
key={level}
onClick={() =>
onSetFilter({
...filter,
levels: {
...filter.levels,
[level]: !enabled,
},
})
}
>
<Tag
key={level}
intent={enabled ? 'primary' : 'none'}
interactive
minimal={!enabled}
round
>
{level.toLowerCase()}
</Tag>
</FilterButton>
<FilterLabel key={level} $enabled={enabled}>
<Checkbox
format="switch"
size="small"
checked={!!enabled}
fillColor={enabled ? Colors.Blue500 : Colors.Gray200}
onChange={() =>
onSetFilter({
...filter,
levels: {
...filter.levels,
[level]: !enabled,
},
})
}
/>
<LogLabel $enabled={enabled}>{level.toLowerCase()}</LogLabel>
<LogCount $enabled={enabled}>{compactNumber(counts[level])}</LogCount>
</FilterLabel>
);
})}
</Group>
</Box>
{selectedStep && <OptionsDivider />}
<div style={{minWidth: 15, flex: 1}} />
<Button
Expand All @@ -357,15 +366,47 @@ const NonMatchCheckbox = styled(Checkbox)`
white-space: nowrap;
`;

const FilterButton = styled.button`
background: none;
const FilterLabel = styled.label<{$enabled: boolean}>`
background-color: ${({$enabled}) => ($enabled ? Colors.Blue50 : Colors.Gray100)};
border: none;
padding: 0;
border-radius: 8px;
margin: 0;
padding: 4px 6px;
overflow: hidden;
cursor: pointer;
display: block;
display: inline-flex;
flex-direction: row;
align-items: center;
font-size: 12px;
font-weight: 500;
gap: 6px;
box-shadow: transparent inset 0px 0px 0px 1px;
transition: background 50ms linear;
:focus {
box-shadow: rgba(58, 151, 212, 0.6) 0 0 0 3px;
outline: none;
}
:focus:not(:focus-visible) {
box-shadow: transparent inset 0px 0px 0px 1px, rgba(0, 0, 0, 0.12) 0px 2px 12px 0px;
}
`;

const LogLabel = styled.span<{$enabled: boolean}>`
color: ${({$enabled}) => ($enabled ? Colors.Blue500 : Colors.Gray700)};
line-height: 16px;
transition: background 50ms linear;
`;

const LogCount = styled.span<{$enabled: boolean}>`
background-color: ${({$enabled}) => ($enabled ? Colors.Blue100 : Colors.Gray200)};
border-radius: 6px;
color: ${({$enabled}) => ($enabled ? Colors.Blue500 : Colors.Gray700)};
font-weight: 600;
font-family: ${FontFamily.monospace};
line-height: 16px;
padding: 1px 4px;
transition: background 50ms linear;
`;
1 change: 1 addition & 0 deletions js_modules/dagit/packages/core/src/runs/Run.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ const RunWithData: React.FC<RunWithDataProps> = ({
computeLogKey={computeLogKey}
onSetComputeLogKey={onSetComputeLogKey}
computeLogUrl={computeLogUrl}
counts={logs.counts}
/>
{logType !== LogType.structured ? (
<ComputeLogPanel
Expand Down
5 changes: 5 additions & 0 deletions js_modules/dagit/packages/core/src/runs/logNodeLevel.tsx
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import {LogLevel} from './LogLevel';
import {LogNode} from './types';

export const logNodeLevel = (node: LogNode): LogLevel =>
node.__typename === 'LogMessageEvent' ? node.level : LogLevel.EVENT;

0 comments on commit 1d216c5

Please sign in to comment.