Skip to content

Commit 109218a

Browse files
kartikgupta-dbkartikgupta-db
authored
Fix flaky integration tests and add test logging (#70)
* integration test fixes * retry job with yarn reinstall * retry job with yarn reinstall * retry job with yarn reinstall * change retry * change retry * change retry * screenshotter * remove console.log * upload test logs * always upload logs * better test log naming * remove vscode binary before integration test * Path p-cancelable * Revert "Path p-cancelable" This reverts commit f1cfafa. * Got patch * Revert "Got patch" This reverts commit f383ae1. * added async lock * don't fail all test when some environments fail * don't fail all test when some environments fail * don't fail all test when some environments fail * don't fail all test when some environments fail * don't fail all test when some environments fail * remove async lock * typo fix * remove 2 second wait * fix periodic runner
1 parent 3a54719 commit 109218a

File tree

12 files changed

+440
-39
lines changed

12 files changed

+440
-39
lines changed

‎.github/workflows/push.yml‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,16 @@ jobs:
1515
matrix:
1616
os: [windows-latest, ubuntu-latest]
1717
node-version: [14.x, 18.x]
18+
fail-fast: false
1819

1920
defaults:
2021
run:
2122
shell: bash
2223
working-directory: packages/databricks-sdk-js
23-
24+
env:
25+
DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
26+
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
27+
TEST_DEFAULT_CLUSTER_ID: ${{ secrets.TEST_DEFAULT_CLUSTER_ID }}
2428
steps:
2529
- uses: actions/checkout@v3
2630

@@ -41,12 +45,14 @@ jobs:
4145
- name: Unit Tests With Code Cov
4246
run: yarn test:cov
4347

44-
- name: Integration Tests With Code Cov
45-
run: yarn run test:integ:cov
46-
env:
47-
DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
48-
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
49-
TEST_DEFAULT_CLUSTER_ID: ${{ secrets.TEST_DEFAULT_CLUSTER_ID }}
48+
- name: Integration Tests
49+
uses: nick-fields/retry@v2
50+
with:
51+
max_attempts: 3
52+
retry_wait_seconds: 2
53+
timeout_minutes: 10
54+
retry_on: any
55+
command: yarn run test:integ:cov
5056

5157
- run: |
5258
cd coverage
@@ -66,6 +72,7 @@ jobs:
6672
os: [windows-latest, macos-latest]
6773
node-version: [16.x]
6874
vscode-version: [stable, insiders]
75+
fail-fast: false
6976

7077
env:
7178
VSCODE_TEST_VERSION: ${{ matrix.vscode-version }}
@@ -104,6 +111,7 @@ jobs:
104111
working-directory: packages/databricks-vscode
105112

106113
- name: Show coverage test result
114+
continue-on-error: true
107115
run: yarn nyc report --reporter text -t ./coverage >> $GITHUB_STEP_SUMMARY
108116
working-directory: packages/databricks-vscode
109117

@@ -112,12 +120,22 @@ jobs:
112120
with:
113121
max_attempts: 3
114122
retry_wait_seconds: 2
115-
timeout_minutes: 2
116-
retry_on: error
123+
timeout_minutes: 10
124+
retry_on: any
117125
command: |
126+
yarn workspace databricks run test:integ:clean
127+
yarn install --immutable
128+
yarn build
118129
cd packages/databricks-vscode
119-
yarn run test:integ:prepare
120-
yarn run test:integ:run
130+
yarn run test:integ
131+
132+
- name: Upload test logs
133+
if: always()
134+
continue-on-error: true
135+
uses: actions/upload-artifact@v3
136+
with:
137+
name: test-logs ${{ join(matrix.*, ' - ') }} - ${{ github.event_name }}
138+
path: packages/databricks-vscode/src/test/logs
121139

122140
package:
123141
name: Package VSIX

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ out/
1212
!.yarn/sdks
1313
!.yarn/versions
1414
**/coverage/
15-
**/.nyc_output/
15+
**/.nyc_output/
16+
.databricks/**

‎packages/databricks-sdk-js/src/retries/Time.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,22 @@ export default class Time {
1717
}
1818

1919
public toMillSeconds(): Time {
20-
let secondsValue = 0;
20+
let milliSecondsValue = 0;
2121
switch (this.units) {
2222
case TimeUnits.hours:
23-
secondsValue = this.value * 60 * 60 * 1000;
23+
milliSecondsValue = this.value * 60 * 60 * 1000;
2424
break;
2525
case TimeUnits.minutes:
26-
secondsValue = this.value * 60 * 1000;
26+
milliSecondsValue = this.value * 60 * 1000;
2727
break;
2828
case TimeUnits.seconds:
29-
secondsValue = this.value * 1000;
29+
milliSecondsValue = this.value * 1000;
3030
break;
3131
case TimeUnits.milliseconds:
32-
secondsValue = this.value;
32+
milliSecondsValue = this.value;
3333
break;
3434
}
35-
return new Time(secondsValue, TimeUnits.seconds);
35+
return new Time(milliSecondsValue, TimeUnits.milliseconds);
3636
}
3737

3838
public add(other: Time): Time {

‎packages/databricks-vscode/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
**/logs/**
12
bin/**

‎packages/databricks-vscode/package.json‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -282,9 +282,11 @@
282282
"fix": "prettier . --write",
283283
"test:lint": "eslint src --ext ts && prettier . -c",
284284
"test:unit": "yarn run build && node ./out/test/runTest.js",
285-
"test:integ:prepare": "yarn run package && extest get-vscode --code_version 1.69.1 --storage /tmp/vscode-test-databricks && extest get-chromedriver --storage /tmp/vscode-test-databricks && extest install-vsix -f databricks-0.0.1.vsix --storage /tmp/vscode-test-databricks",
286-
"test:integ:run": "yarn run build && node out/test/e2e/scripts/e2e.js --storage /tmp/vscode-test-databricks --code_settings src/test/e2e/settings.json 'out/**/*.e2e.js'",
287-
"test:integ": "yarn run test:integ:prepare && yarn run test:integ:run",
285+
"test:integ:clean": "yarn run clean && rm -rf /tmp/vscode-test-databricks /tmp/databricks-vscode-test-extensions",
286+
"test:integ:prepare": "yarn run package && extest get-vscode --code_version 1.69.1 --storage /tmp/vscode-test-databricks && extest get-chromedriver --storage /tmp/vscode-test-databricks",
287+
"test:integ:install-vsix": "extest install-vsix --storage /tmp/vscode-test-databricks -f databricks-0.0.1.vsix -e /tmp/databricks-vscode-test-extensions",
288+
"test:integ:run": "yarn run build && node out/test/e2e/scripts/e2e.js -e /tmp/databricks-vscode-test-extensions --storage /tmp/vscode-test-databricks --code_settings src/test/e2e/settings.json 'out/**/*.e2e.js'",
289+
"test:integ": "yarn run test:integ:prepare && yarn run test:integ:install-vsix && yarn run test:integ:run",
288290
"test:cov": "nyc yarn run test:unit",
289291
"test": "yarn run test:lint && yarn run test:unit",
290292
"clean": "rm -rf node_modules out .vscode-test"
@@ -313,7 +315,8 @@
313315
"ts-mockito": "^2.6.1",
314316
"typescript": "^4.7.4",
315317
"vsce": "^2.10.0",
316-
"vscode-extension-tester": "^4.4.1"
318+
"vscode-extension-tester": "^4.4.1",
319+
"winston": "^3.8.1"
317320
},
318321
"nyc": {
319322
"extends": "@istanbuljs/nyc-config-typescript",

‎packages/databricks-vscode/src/test/PeriodicRunner.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import Time, {TimeUnits} from "@databricks/databricks-sdk/dist/retries/Time";
2+
3+
export interface RunFunctionDetails {
4+
fn: () => Promise<void>;
5+
cleanup?: () => Promise<void>;
6+
every: Time;
7+
timer?: NodeJS.Timer;
8+
}
9+
10+
export class PeriodicRunner {
11+
private runFunctions: RunFunctionDetails[] = [];
12+
13+
runFunction(runFunctionDetails: RunFunctionDetails) {
14+
this.runFunctions.push(runFunctionDetails);
15+
return this;
16+
}
17+
18+
start() {
19+
this.runFunctions.forEach((runFunction, idx) => {
20+
this.runFunctions[idx].timer = setInterval(
21+
runFunction.fn,
22+
runFunction.every.toMillSeconds().value
23+
);
24+
});
25+
}
26+
27+
async stop() {
28+
this.runFunctions.forEach((runFunction) => {
29+
if (runFunction.timer) {
30+
clearInterval(runFunction.timer);
31+
}
32+
});
33+
34+
this.runFunctions.forEach(async (runFunction) => {
35+
if (runFunction.cleanup) {
36+
await runFunction.cleanup();
37+
}
38+
});
39+
}
40+
}

‎packages/databricks-vscode/src/test/e2e/configure.e2e.ts‎

Lines changed: 53 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import assert from "node:assert";
2-
import path from "node:path";
2+
import path, {resolve} from "node:path";
33
import * as fs from "fs/promises";
44
import * as tmp from "tmp-promise";
55
import {
@@ -12,8 +12,11 @@ import {
1212
CustomTreeSection,
1313
} from "vscode-extension-tester";
1414
import {getViewSection, openFolder, waitForTreeItems} from "./utils";
15+
import Time, {TimeUnits} from "@databricks/databricks-sdk/dist/retries/Time";
16+
import {PeriodicRunner} from "../PeriodicRunner";
17+
import {ImageLogger, Logger} from "../loggingUtils";
1518

16-
describe("Configure Databricks Extension", function () {
19+
describe("Configure Databricks Extension", async function () {
1720
// these will be populated by the before() function
1821
let browser: VSBrowser;
1922
let driver: WebDriver;
@@ -22,10 +25,11 @@ describe("Configure Databricks Extension", function () {
2225

2326
// this will be populated by the tests
2427
let clusterId: string;
28+
let periodicRunners = new Map<string, PeriodicRunner>();
2529

2630
this.timeout(10 * 60 * 1000);
2731

28-
before(async () => {
32+
before(async function () {
2933
browser = VSBrowser.instance;
3034
driver = browser.driver;
3135

@@ -36,16 +40,54 @@ describe("Configure Databricks Extension", function () {
3640
await openFolder(browser, projectDir);
3741
});
3842

39-
after(() => {
43+
beforeEach(async function () {
44+
const testName = this.currentTest?.title ?? "Default";
45+
const suiteName = this.currentTest?.parent?.title ?? "Default";
46+
47+
const logger = await Logger.getLogger(suiteName, testName);
48+
const imageLogger = ImageLogger.getLogger(suiteName, testName);
49+
50+
const periodicRunner = new PeriodicRunner()
51+
.runFunction({
52+
fn: async () => {
53+
(await driver.manage().logs().get("browser")).map(
54+
(entry) => {
55+
logger.info(entry.message);
56+
}
57+
);
58+
},
59+
every: new Time(1, TimeUnits.seconds),
60+
})
61+
.runFunction({
62+
fn: async () => {
63+
await imageLogger.log(await driver.takeScreenshot());
64+
},
65+
cleanup: async () => {
66+
await imageLogger.flush();
67+
},
68+
every: new Time(1, TimeUnits.seconds),
69+
});
70+
71+
periodicRunner.start();
72+
periodicRunners.set(testName, periodicRunner);
73+
});
74+
75+
afterEach(async function () {
76+
const testName = this.currentTest?.title ?? "Default";
77+
await periodicRunners.get(testName)?.stop();
78+
periodicRunners.delete(testName);
79+
});
80+
81+
after(function () {
4082
cleanup();
4183
});
4284

43-
it("should open VSCode", async () => {
85+
it("should open VSCode", async function () {
4486
const title = await driver.getTitle();
4587
assert(title.indexOf("Get Started") >= 0);
4688
});
4789

48-
it("should open databricks panel and login", async () => {
90+
it("should open databricks panel and login", async function () {
4991
const section = await getViewSection("Configuration");
5092
assert(section);
5193
const welcome = await section.findWelcomeContent();
@@ -61,14 +103,14 @@ describe("Configure Databricks Extension", function () {
61103
assert(await waitForTreeItems(section));
62104
});
63105

64-
it("should dismiss notifications", async () => {
106+
it("should dismiss notifications", async function () {
65107
const notifications = await new Workbench().getNotifications();
66108
for (const n of notifications) {
67109
await n.dismiss();
68110
}
69111
});
70112

71-
it("shoult list clusters", async () => {
113+
it("shoult list clusters", async function () {
72114
const section = await getViewSection("Clusters");
73115
assert(section);
74116
const tree = section as CustomTreeSection;
@@ -82,7 +124,7 @@ describe("Configure Databricks Extension", function () {
82124

83125
// test is skipped because context menus currently don't work in vscode-extension-tester
84126
// https://github.com/redhat-developer/vscode-extension-tester/issues/444
85-
it.skip("should filter clusters", async () => {
127+
it.skip("should filter clusters", async function () {
86128
const section = await getViewSection("Clusters");
87129
assert(section);
88130
const action = await section!.getAction("Filter clusters ...");
@@ -97,7 +139,7 @@ describe("Configure Databricks Extension", function () {
97139
assert(items.length > 0);
98140
});
99141

100-
it("should attach cluster", async () => {
142+
it("should attach cluster", async function () {
101143
const config = await getViewSection("Configuration");
102144
assert(config);
103145
const configTree = config as CustomTreeSection;
@@ -128,7 +170,6 @@ describe("Configure Databricks Extension", function () {
128170

129171
await input.setText(clusterId);
130172
await input.confirm();
131-
await input.selectQuickPick(0);
132173

133174
// get cluster ID
134175
const clusterPropsItems = await clusterConfigItem.getChildren();
@@ -142,7 +183,7 @@ describe("Configure Databricks Extension", function () {
142183
assert(clusterId);
143184
});
144185

145-
it("should write the project config file", async () => {
186+
it("should write the project config file", async function () {
146187
let projectConfig = JSON.parse(
147188
await fs.readFile(
148189
path.join(projectDir, ".databricks", "project.json"),

‎packages/databricks-vscode/src/test/e2e/scripts/e2e.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,15 @@ async function main(args: string[]) {
2222
);
2323

2424
const {path: configFile, cleanup} = await tmp.file();
25+
let host = process.env["DATABRICKS_HOST"];
26+
if (!host.startsWith("http")) {
27+
host = `https://${host}`;
28+
}
2529
try {
2630
await fs.writeFile(
2731
configFile,
2832
`[DEFAULT]
29-
host = ${process.env["DATABRICKS_HOST"]}
33+
host = ${host}
3034
token = ${process.env["DATABRICKS_TOKEN"]}`
3135
);
3236

0 commit comments

Comments
 (0)