Add e2e tests for bundle init flow (#1073)

Add smoke tests for bundle init and auth flows.

Also fixes these problems:
- DATABRICKS_CONFIG_FILE env var was not always respected (for example
when creating a new profile)
- bundle file watcher was broken on windows: we were using raw fsPath
with backslashes, which doesn't work for globs
- saveNewProfile was broken on windows: `toISOString` returns values
with characters that aren't allowed in windows paths, resulting in
`ENOENT` errors from `copyFile`

Author: ilia-db

.eslintrc.json

@@ -26,7 +26,7 @@
     },
     "overrides": [
         {
-            "files": "**/*.test.ts",
+            "files": ["**/*.test.ts", "**/test/**"],
             "rules": {
                 "no-console": "off"
             }

.github/workflows/tests.yml

@@ -76,9 +76,9 @@ jobs:
           run: yarn run test:cov
           working-directory: packages/databricks-vscode
 
-      # - name: Integration Tests
-      #   run: yarn run test:integ
-      #   working-directory: packages/databricks-vscode
+      - name: Integration Tests
+        run: yarn run test:integ
+        working-directory: packages/databricks-vscode
 
       - name: Integration Tests SDK wrappers
         run: yarn run test:integ:sdk

packages/databricks-vscode/package.json

@@ -6,7 +6,7 @@
     "license": "LicenseRef-LICENSE",
     "version": "2.0.0",
     "engines": {
-        "vscode": "^1.83.0"
+        "vscode": "^1.86.0"
     },
     "categories": [
         "Data Science",
@@ -844,8 +844,8 @@
         "ts-node": "^10.9.2",
         "typescript": "^5.3.3",
         "vsce": "^2.15.0",
-        "wdio-video-reporter": "^4.0.5",
-        "wdio-vscode-service": "^5.2.2",
+        "wdio-video-reporter": "^5.1.4",
+        "wdio-vscode-service": "^6.0.2",
         "yargs": "^17.7.2"
     },
     "nyc": {

packages/databricks-vscode/src/bundle/BundleFileSet.test.ts

@@ -1,5 +1,5 @@
 import {Uri} from "vscode";
-import {BundleFileSet, getAbsolutePath} from "./BundleFileSet";
+import {BundleFileSet, getAbsoluteGlobPath} from "./BundleFileSet";
 import {expect} from "chai";
 import path from "path";
 import * as tmp from "tmp-promise";
@@ -18,16 +18,18 @@ describe(__filename, async function () {
         await tmpdir.cleanup();
     });
 
-    it("should return the correct absolute path", () => {
+    it("should return the correct absolute glob path", () => {
         const tmpdirUri = Uri.file(tmpdir.path);
-
-        expect(getAbsolutePath("test.txt", tmpdirUri).fsPath).to.equal(
-            path.join(tmpdirUri.fsPath, "test.txt")
+        let expectedGlob = path.join(tmpdirUri.fsPath, "test.txt");
+        if (process.platform === "win32") {
+            expectedGlob = expectedGlob.replace(/\\/g, "/");
+        }
+        expect(getAbsoluteGlobPath("test.txt", tmpdirUri)).to.equal(
+            expectedGlob
+        );
+        expect(getAbsoluteGlobPath(Uri.file("test.txt"), tmpdirUri)).to.equal(
+            expectedGlob
         );
-
-        expect(
-            getAbsolutePath(Uri.file("test.txt"), tmpdirUri).fsPath
-        ).to.equal(path.join(tmpdirUri.fsPath, "test.txt"));
     });
 
     it("should find the correct root bundle yaml", async () => {

packages/databricks-vscode/src/bundle/BundleFileSet.ts

@@ -22,7 +22,7 @@ export async function writeBundleYaml(file: Uri, data: BundleSchema) {
 
 export async function getSubProjects(root: Uri) {
     const subProjectRoots = await glob.glob(
-        toGlobPath(getAbsolutePath(subProjectFilePattern, root).fsPath),
+        getAbsoluteGlobPath(subProjectFilePattern, root),
         {nocase: process.platform === "win32"}
     );
     const normalizedRoot = path.normalize(root.fsPath);
@@ -40,11 +40,10 @@ export async function getSubProjects(root: Uri) {
         });
 }
 
-export function getAbsolutePath(path: string | Uri, root: Uri) {
-    if (typeof path === "string") {
-        return Uri.joinPath(root, path);
-    }
-    return Uri.joinPath(root, path.fsPath);
+export function getAbsoluteGlobPath(path: string | Uri, root: Uri): string {
+    path = typeof path === "string" ? path : path.fsPath;
+    const uri = Uri.joinPath(root, path);
+    return toGlobPath(uri.fsPath);
 }
 
 function toGlobPath(path: string) {
@@ -68,9 +67,7 @@ export class BundleFileSet {
 
     async getRootFile() {
         const rootFile = await glob.glob(
-            toGlobPath(
-                getAbsolutePath(rootFilePattern, this.workspaceRoot).fsPath
-            ),
+            getAbsoluteGlobPath(rootFilePattern, this.workspaceRoot),
             {nocase: process.platform === "win32"}
         );
         if (rootFile.length !== 1) {
@@ -83,11 +80,9 @@ export class BundleFileSet {
         root?: Uri
     ): Promise<{relative: Uri; absolute: Uri}[]> {
         const subProjectRoots = await glob.glob(
-            toGlobPath(
-                getAbsolutePath(
-                    subProjectFilePattern,
-                    root || this.workspaceRoot
-                ).fsPath
+            getAbsoluteGlobPath(
+                subProjectFilePattern,
+                root || this.workspaceRoot
             ),
             {nocase: process.platform === "win32"}
         );
@@ -167,9 +162,7 @@ export class BundleFileSet {
     isRootBundleFile(e: Uri) {
         return minimatch(
             e.fsPath,
-            toGlobPath(
-                getAbsolutePath(rootFilePattern, this.workspaceRoot).fsPath
-            )
+            getAbsoluteGlobPath(rootFilePattern, this.workspaceRoot)
         );
     }
 
@@ -178,10 +171,10 @@ export class BundleFileSet {
         if (includedFilesGlob === undefined) {
             return false;
         }
-        includedFilesGlob = getAbsolutePath(
+        includedFilesGlob = getAbsoluteGlobPath(
             includedFilesGlob,
             this.workspaceRoot
-        ).fsPath;
+        );
         return minimatch(e.fsPath, toGlobPath(includedFilesGlob));
     }
 

packages/databricks-vscode/src/bundle/BundleWatcher.ts

@@ -1,5 +1,5 @@
 import {Disposable, EventEmitter, Uri, workspace} from "vscode";
-import {BundleFileSet, getAbsolutePath} from "./BundleFileSet";
+import {BundleFileSet, getAbsoluteGlobPath} from "./BundleFileSet";
 import {WithMutex} from "../locking";
 import path from "path";
 
@@ -23,8 +23,7 @@ export class BundleWatcher implements Disposable {
     constructor(bundleFileSet: BundleFileSet, workspaceUri: Uri) {
         this.bundleFileSet = new WithMutex(bundleFileSet);
         const yamlWatcher = workspace.createFileSystemWatcher(
-            getAbsolutePath(path.join("**", "*.{yaml,yml}"), workspaceUri)
-                .fsPath
+            getAbsoluteGlobPath(path.join("**", "*.{yaml,yml}"), workspaceUri)
         );
 
         this.disposables.push(

packages/databricks-vscode/src/configuration/ConnectionCommands.ts

@@ -12,6 +12,9 @@ import {ConnectionManager} from "./ConnectionManager";
 import {UrlUtils} from "../utils";
 import {WorkspaceFsCommands} from "../workspace-fs";
 import {ConfigModel} from "./models/ConfigModel";
+import {saveNewProfile} from "./LoginWizard";
+import {PersonalAccessTokenAuthProvider} from "./auth/AuthProvider";
+import {normalizeHost} from "../utils/urlUtils";
 
 function formatQuickPickClusterSize(sizeInMB: number): string {
     if (sizeInMB > 1024) {
@@ -73,6 +76,19 @@ export class ConnectionCommands implements Disposable {
         );
     }
 
+    // This command is not exposed to users.
+    // We use it to test new profile flow in e2e tests.
+    async saveNewProfileCommand(name: string) {
+        const host = this.connectionManager.workspaceClient?.config.host;
+        const token = this.connectionManager.workspaceClient?.config.token;
+        if (!host || !token) {
+            throw new Error("Must login first");
+        }
+        const hostUrl = normalizeHost(host);
+        const provider = new PersonalAccessTokenAuthProvider(hostUrl, token);
+        await saveNewProfile(name, provider);
+    }
+
     /**
      * Attach to cluster from settings. If attach fails or no cluster is configured
      * then show dialog to select (or create) a cluster. The selected cluster is saved

packages/databricks-vscode/src/configuration/LoginWizard.ts

@@ -387,7 +387,7 @@ export async function saveNewProfile(
     // Create a backup for .databrickscfg
     const backup = path.join(
         path.dirname(configFilePath),
-        `.databrickscfg.${new Date().toISOString()}.bak`
+        `.databrickscfg.${Date.now()}.bak`
     );
     await copyFile(configFilePath, backup);
     window.showInformationMessage(

packages/databricks-vscode/src/configuration/auth/AuthProvider.ts

@@ -204,9 +204,7 @@ export class ProfileAuthProvider extends AuthProvider {
     public static getSdkConfig(profile: string): Config {
         return new Config({
             profile: profile,
-            configFile:
-                workspaceConfigs.databrickscfgLocation ??
-                process.env.DATABRICKS_CONFIG_FILE,
+            configFile: workspaceConfigs.databrickscfgLocation,
             env: {},
         });
     }

packages/databricks-vscode/src/extension.ts

@@ -486,6 +486,11 @@ export async function activate(
             "databricks.connection.detachCluster",
             connectionCommands.detachClusterCommand(),
             connectionCommands
+        ),
+        telemetry.registerCommand(
+            "databricks.connection.saveNewProfile",
+            connectionCommands.saveNewProfileCommand,
+            connectionCommands
         )
     );