Skip to content

Commit 3c3fc9c

Browse files
ilia-dbilia-db
authored
Add Event Log section to the pipeline subtree (#1444)
## Changes Running state: <img width="533" alt="Screenshot 2024-11-08 at 15 15 23" src="https://github.com/user-attachments/assets/f518a051-6d1d-4ffb-bf4b-33af7482d1c8"> Completed state: <img width="515" alt="Screenshot 2024-11-08 at 15 15 50" src="https://github.com/user-attachments/assets/9f79d65d-f4de-402c-98b8-5ac447150d45"> Currently event log items aren't interactive. Requires this JS SDK PR to be merged first: databricks/databricks-sdk-js#272 ## Tests Manually
1 parent aa8eae8 commit 3c3fc9c

File tree

5 files changed

+174
-35
lines changed

5 files changed

+174
-35
lines changed

‎packages/databricks-vscode/src/bundle/run/PipelineRunStatus.ts‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import {BundleRunStatus} from "./BundleRunStatus";
33
import {AuthProvider} from "../../configuration/auth/AuthProvider";
44
import {onError} from "../../utils/onErrorDecorator";
5-
import {pipelines} from "@databricks/databricks-sdk";
5+
import {pipelines, WorkspaceClient} from "@databricks/databricks-sdk";
66

77
function isRunning(status?: pipelines.UpdateInfoState) {
88
if (status === undefined) {
@@ -12,17 +12,19 @@ function isRunning(status?: pipelines.UpdateInfoState) {
1212
}
1313

1414
export class PipelineRunStatus extends BundleRunStatus {
15-
readonly type = "pipelines";
15+
public readonly type = "pipelines";
16+
public data: pipelines.GetUpdateResponse | undefined;
17+
public events: pipelines.PipelineEvent[] | undefined;
18+
1619
private interval?: NodeJS.Timeout;
17-
data: pipelines.GetUpdateResponse | undefined;
20+
1821
constructor(
1922
private readonly authProvider: AuthProvider,
2023
private readonly pipelineId: string
2124
) {
2225
super();
2326
}
2427

25-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
2628
parseId(output: string): void {
2729
if (this.runId !== undefined || this.runState !== "unknown") {
2830
return;
@@ -57,26 +59,52 @@ export class PipelineRunStatus extends BundleRunStatus {
5759
if (this.runId === undefined) {
5860
throw new Error("No update id");
5961
}
60-
const update = await client.pipelines.getUpdate({
62+
this.data = await client.pipelines.getUpdate({
6163
pipeline_id: this.pipelineId,
6264
update_id: this.runId,
6365
});
64-
this.data = update;
66+
67+
if (this.data.update?.creation_time !== undefined) {
68+
this.events = await this.fetchUpdateEvents(
69+
client,
70+
this.data.update.creation_time,
71+
this.data.update?.update_id
72+
);
73+
}
6574

6675
// If update is completed, we stop polling.
67-
if (!isRunning(update.update?.state)) {
76+
if (!isRunning(this.data.update?.state)) {
6877
this.markCompleted();
69-
return;
78+
} else {
79+
this.onDidChangeEmitter.fire();
7080
}
71-
72-
this.onDidChangeEmitter.fire();
7381
} catch (e) {
7482
this.runState = "error";
7583
throw e;
7684
}
7785
}, 5_000);
7886
}
7987

88+
private async fetchUpdateEvents(
89+
client: WorkspaceClient,
90+
creationTime: number,
91+
updateId?: string
92+
) {
93+
const events = [];
94+
const timestamp = new Date(creationTime).toISOString();
95+
const listEvents = client.pipelines.listPipelineEvents({
96+
pipeline_id: this.pipelineId,
97+
order_by: ["timestamp asc"],
98+
filter: `timestamp >= '${timestamp}'`,
99+
});
100+
for await (const event of listEvents) {
101+
if (!updateId || event.origin?.update_id === updateId) {
102+
events.push(event);
103+
}
104+
}
105+
return events;
106+
}
107+
80108
private markCompleted() {
81109
if (this.interval !== undefined) {
82110
clearInterval(this.interval);

‎packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunEventsTreeNode.ts‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import {
2+
BundleResourceExplorerTreeItem,
3+
BundleResourceExplorerTreeNode,
4+
} from "./types";
5+
import {ThemeColor, ThemeIcon, TreeItemCollapsibleState} from "vscode";
6+
import {ContextUtils} from "./utils";
7+
import {PipelineRunStatus} from "../../bundle/run/PipelineRunStatus";
8+
import {TreeItemTreeNode} from "../TreeItemTreeNode";
9+
import {ConnectionManager} from "../../configuration/ConnectionManager";
10+
import {EventLevel} from "@databricks/databricks-sdk/dist/apis/pipelines";
11+
12+
export class PipelineRunEventsTreeNode
13+
implements BundleResourceExplorerTreeNode
14+
{
15+
readonly type = "pipeline_run_events";
16+
17+
private get update() {
18+
return this.runMonitor?.data?.update;
19+
}
20+
21+
private get events() {
22+
return this.runMonitor?.events;
23+
}
24+
25+
public get url() {
26+
const {host} = this.connectionManager.databricksWorkspace ?? {};
27+
// eslint-disable-next-line @typescript-eslint/naming-convention
28+
const {pipeline_id, update_id} = this.update ?? {};
29+
if (!host || !pipeline_id || !update_id) {
30+
return undefined;
31+
}
32+
return `${host}#joblist/pipelines/${pipeline_id}/updates/${update_id}`;
33+
}
34+
35+
constructor(
36+
private readonly connectionManager: ConnectionManager,
37+
private readonly runMonitor: PipelineRunStatus,
38+
public parent?: BundleResourceExplorerTreeNode
39+
) {}
40+
41+
getChildren(): BundleResourceExplorerTreeNode[] {
42+
if (this.events === undefined || this.events.length === 0) {
43+
return [];
44+
}
45+
const children: BundleResourceExplorerTreeNode[] = [];
46+
47+
for (const event of this.events) {
48+
children.push(
49+
new TreeItemTreeNode(
50+
{
51+
label: event.message ?? event.event_type ?? "unknown",
52+
iconPath: getEventIcon(event.level),
53+
tooltip: event.message,
54+
contextValue: "pipeline_event",
55+
},
56+
this
57+
)
58+
);
59+
}
60+
61+
return children;
62+
}
63+
64+
getTreeItem(): BundleResourceExplorerTreeItem {
65+
if (this.events === undefined || this.events.length === 0) {
66+
return {
67+
label: "Event Log",
68+
iconPath: new ThemeIcon("loading~spin"),
69+
contextValue: ContextUtils.getContextString({
70+
nodeType: this.type,
71+
}),
72+
collapsibleState: TreeItemCollapsibleState.None,
73+
};
74+
}
75+
76+
return {
77+
label: "Event Log",
78+
iconPath: new ThemeIcon("inbox"),
79+
contextValue: ContextUtils.getContextString({
80+
nodeType: this.type,
81+
hasUrl: this.url !== undefined,
82+
}),
83+
collapsibleState: TreeItemCollapsibleState.Expanded,
84+
};
85+
}
86+
}
87+
88+
function getEventIcon(level: EventLevel | undefined): ThemeIcon {
89+
switch (level) {
90+
case "ERROR":
91+
return new ThemeIcon(
92+
"error",
93+
new ThemeColor("list.errorForeground")
94+
);
95+
case "INFO":
96+
return new ThemeIcon("info");
97+
case "METRICS":
98+
return new ThemeIcon("dashboard");
99+
case "WARN":
100+
return new ThemeIcon(
101+
"warning",
102+
new ThemeColor("list.warningForeground")
103+
);
104+
default:
105+
return new ThemeIcon("question");
106+
}
107+
}

‎packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineRunStatusTreeNode.ts‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import {
22
BundleResourceExplorerTreeItem,
33
BundleResourceExplorerTreeNode,
44
} from "./types";
5-
import {TreeItemCollapsibleState} from "vscode";
5+
import {ThemeIcon, TreeItemCollapsibleState} from "vscode";
66
import {ContextUtils, RunStateUtils} from "./utils";
77
import {SimplifiedRunState, sentenceCase} from "./utils/RunStateUtils";
88
import {GetUpdateResponse} from "@databricks/databricks-sdk/dist/apis/pipelines";
@@ -44,24 +44,19 @@ export class PipelineRunStatusTreeNode
4444
implements BundleResourceExplorerTreeNode
4545
{
4646
readonly type = "pipeline_run_status";
47+
4748
private get update() {
4849
return this.runMonitor?.data?.update;
4950
}
51+
5052
public get url() {
51-
if (this.type !== this.type) {
52-
return undefined;
53-
}
54-
const host = this.connectionManager.databricksWorkspace?.host;
55-
if (
56-
host === undefined ||
57-
this.update?.pipeline_id === undefined ||
58-
this.update?.update_id === undefined
59-
) {
53+
const {host} = this.connectionManager.databricksWorkspace ?? {};
54+
// eslint-disable-next-line @typescript-eslint/naming-convention
55+
const {pipeline_id, update_id} = this.update ?? {};
56+
if (!host || !pipeline_id || !update_id) {
6057
return undefined;
6158
}
62-
return `${host.toString()}#joblist/pipelines/${
63-
this.update.pipeline_id
64-
}/updates/${this.update.update_id}`;
59+
return `${host}#joblist/pipelines/${pipeline_id}/updates/${update_id}`;
6560
}
6661

6762
constructor(
@@ -122,8 +117,7 @@ export class PipelineRunStatusTreeNode
122117
if (this.update === undefined) {
123118
return {
124119
label: "Run Status",
125-
iconPath: RunStateUtils.getThemeIconForStatus("Unknown"),
126-
description: "Run status not available",
120+
iconPath: new ThemeIcon("loading~spin"),
127121
contextValue: ContextUtils.getContextString({
128122
nodeType: this.type,
129123
}),

‎packages/databricks-vscode/src/ui/bundle-resource-explorer/PipelineTreeNode.ts‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import {ConnectionManager} from "../../configuration/ConnectionManager";
1212
import {PipelineRunStatus} from "../../bundle/run/PipelineRunStatus";
1313
import {TreeItemTreeNode} from "../TreeItemTreeNode";
1414
import {PipelineRunStatusTreeNode} from "./PipelineRunStatusTreeNode";
15+
import {PipelineRunEventsTreeNode} from "./PipelineRunEventsTreeNode";
16+
import {ThemeIcon} from "vscode";
1517

1618
export class PipelineTreeNode implements BundleResourceExplorerTreeNode {
1719
readonly type = "pipelines";
@@ -67,22 +69,13 @@ export class PipelineTreeNode implements BundleResourceExplorerTreeNode {
6769
const runMonitor = this.bundleRunStatusManager.runStatuses.get(
6870
this.resourceKey
6971
) as PipelineRunStatus | undefined;
70-
if (runMonitor) {
71-
children.push(
72-
new PipelineRunStatusTreeNode(
73-
this.connectionManager,
74-
runMonitor,
75-
this
76-
)
77-
);
78-
}
79-
8072
if (this.data.catalog) {
8173
children.push(
8274
new TreeItemTreeNode(
8375
{
8476
label: "Catalog",
8577
description: this.data.catalog,
78+
iconPath: new ThemeIcon("book"),
8679
contextValue: "catalog",
8780
},
8881
this
@@ -96,13 +89,29 @@ export class PipelineTreeNode implements BundleResourceExplorerTreeNode {
9689
{
9790
label: "Target",
9891
description: this.data.target,
92+
iconPath: new ThemeIcon("target"),
9993
contextValue: "target",
10094
},
10195
this
10296
)
10397
);
10498
}
10599

100+
if (runMonitor) {
101+
children.push(
102+
new PipelineRunStatusTreeNode(
103+
this.connectionManager,
104+
runMonitor,
105+
this
106+
),
107+
new PipelineRunEventsTreeNode(
108+
this.connectionManager,
109+
runMonitor,
110+
this
111+
)
112+
);
113+
}
114+
106115
return children;
107116
}
108117

‎packages/databricks-vscode/src/ui/bundle-resource-explorer/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ export interface BundleResourceExplorerTreeNode {
1515
| "treeItem"
1616
| "task_run_status"
1717
| "pipeline_run_status"
18+
| "pipeline_run_events"
1819
| "resource_type_header"
1920
| "task"
2021
| "job_run_status"

0 commit comments

Comments
 (0)