Fix multiple quote issue for .databricks.env file. (#686)

## Changes
* Variables being added to .databricks.env were being added with
multiple quotes. This was because of the following sequence of events:
* Vscode would try to search for user env file (.env) and not find it.
* It would then try to search for mspython python.envFile and not find
it.
* Finally it will set the userEnvFile to .env and the mspython
python.envFile to .databricks/.databricks.env.
* At the same time, if another instance of vscode is running in the same
project, it will go through the same steps, but this time find mspython
python.envFile to be .databricks/.databricks.env. It will use this value
for userEnvFile.
* Now both values are set to .databricks/.databricks.env. This is an
issue.

This is fixed by
* Stripping quotes
* More strict checking for .databricks/.databricks.env appearing in
userEnvFile.

This pr also adds an integration test for running .ipynb as jobs.

---------

Signed-off-by: kartikgupta-db <kartik.gupta@databricks.com>

Author: kartikgupta-db

packages/databricks-vscode/package.json

@@ -373,7 +373,7 @@
                 },
                 {
                     "command": "databricks.run.runEditorContentsAsWorkflow",
-                    "when": "resourceLangId == python"
+                    "when": "resourceLangId == python || resourceExtname == .ipynb"
                 },
                 {
                     "command": "databricks.wsfs.createFolder",

packages/databricks-vscode/src/file-managers/DatabricksEnvFileManager.ts

@@ -20,6 +20,7 @@ import {logging} from "@databricks/databricks-sdk";
 import {Loggers} from "../logger";
 import {Context, context} from "@databricks/databricks-sdk/dist/context";
 import {MsPythonExtensionWrapper} from "../language/MsPythonExtensionWrapper";
+import {NamedLogger} from "@databricks/databricks-sdk/dist/logging";
 
 export class DatabricksEnvFileManager implements Disposable {
     private disposables: Disposable[] = [];
@@ -48,18 +49,27 @@ export class DatabricksEnvFileManager implements Disposable {
             systemVariableResolver.resolve(unresolvedDatabricksEnvFile)
         );
 
-        const unresolvedUserEnvFile = workspaceConfigs.userEnvFile
+        const unresolvedUserEnvFile = this.isValidUserEnvPath(
+            workspaceConfigs.userEnvFile,
+            [unresolvedDatabricksEnvFile, this.databricksEnvPath.fsPath]
+        )
             ? workspaceConfigs.userEnvFile
-            : !workspaceConfigs.msPythonEnvFile ||
-              workspaceConfigs.msPythonEnvFile.includes(
-                  this.databricksEnvPath.fsPath
-              )
-            ? path.join("${workspaceFolder}", ".env")
-            : workspaceConfigs.msPythonEnvFile;
+            : this.isValidUserEnvPath(workspaceConfigs.msPythonEnvFile, [
+                  unresolvedDatabricksEnvFile,
+                  this.databricksEnvPath.fsPath,
+              ])
+            ? workspaceConfigs.msPythonEnvFile
+            : path.join("${workspaceFolder}", ".env");
         this.userEnvPath = Uri.file(
             systemVariableResolver.resolve(unresolvedUserEnvFile)
         );
 
+        NamedLogger.getOrCreate(Loggers.Extension).debug("Env file locations", {
+            unresolvedDatabricksEnvFile,
+            unresolvedUserEnvFile,
+            msEnvFile: workspaceConfigs.msPythonEnvFile,
+        });
+
         workspaceConfigs.msPythonEnvFile = unresolvedDatabricksEnvFile;
         workspaceConfigs.userEnvFile = unresolvedUserEnvFile;
 
@@ -125,6 +135,13 @@ export class DatabricksEnvFileManager implements Disposable {
         );
     }
 
+    private isValidUserEnvPath(
+        path: string | undefined,
+        excludes: string[]
+    ): path is string {
+        return path !== undefined && !excludes.includes(path);
+    }
+
     private async disableStatusBarButton() {
         const featureState = await this.featureManager.isEnabled(
             "debugging.dbconnect"
@@ -263,7 +280,10 @@ export class DatabricksEnvFileManager implements Disposable {
         const data = Object.entries({
             ...databricksEnvVars,
             ...userEnvVars,
-        }).map(([key, value]) => `${key}="${value}"`);
+        }).map(([key, value]) => {
+            value = value?.replaceAll(/^"|"$/g, "");
+            return `${key}="${value}"`;
+        });
         try {
             const oldData = await readFile(
                 this.databricksEnvPath.fsPath,

packages/databricks-vscode/src/test/e2e/run_job_on_cluster_with_workspace.e2e.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
 import * as fs from "fs/promises";
 import assert from "node:assert";
@@ -56,6 +57,30 @@ describe("Run job on cluster with workspace", async function () {
             ].join("\n")
         );
 
+        await fs.writeFile(
+            path.join(projectDir, "notebook_ipynb.ipynb"),
+            JSON.stringify({
+                cells: [
+                    {
+                        cell_type: "code",
+                        execution_count: null,
+                        metadata: {},
+                        outputs: [],
+                        source: ['print("hello")'],
+                    },
+                ],
+                metadata: {
+                    kernelspec: {
+                        display_name: "Python 3",
+                        language: "python",
+                        name: "python3",
+                    },
+                    orig_nbformat: 4,
+                },
+                nbformat: 4,
+                nbformat_minor: 2,
+            })
+        );
         const section = await getViewSection("CONFIGURATION");
         assert(section);
         await waitForTreeItems(section);
@@ -129,6 +154,72 @@ describe("Run job on cluster with workspace", async function () {
         webView.close();
     });
 
+    it("should run a jupyter notebook as a job on a cluster", async () => {
+        await startSyncIfStopped();
+        await waitForSyncComplete();
+
+        const workbench = await driver.getWorkbench();
+        const editorView = workbench.getEditorView();
+        await editorView.closeAllEditors();
+
+        // open file
+        const input = await workbench.openCommandPrompt();
+        await sleep(200);
+        await input.setText("notebook_ipynb.ipynb");
+        await input.confirm();
+        await sleep(500);
+
+        // run file
+        await workbench.executeQuickPick(
+            "Databricks: Run File as Workflow on Databricks"
+        );
+
+        await dismissNotifications();
+        const webView = await workbench.getWebviewByTitle(/Databricks Job Run/);
+        await webView.open();
+
+        /* eslint-disable @typescript-eslint/naming-convention */
+        const labelToDefaults = {
+            taskRunId: {label: "task-run-id", default: /N\\A/},
+            clusterId: {label: "cluster", default: /N\\A/},
+            startTime: {label: "run-start-time", default: /-/},
+            endTime: {label: "run-end-time", default: /-/},
+            duration: {label: "run-duration", default: /-/},
+            status: {label: "run-status", default: /Synchronizing/},
+        };
+        /* eslint-enable @typescript-eslint/naming-convention */
+
+        // wait for job to get a task id
+        await browser.waitUntil(
+            async () =>
+                (
+                    await browser.getTextByLabel(
+                        labelToDefaults.taskRunId.label
+                    )
+                ).match(labelToDefaults.taskRunId.default) === null,
+            {
+                timeoutMsg: "Job did not start",
+            }
+        );
+
+        expect(
+            await browser.getTextByLabel(labelToDefaults.startTime.label)
+        ).not.toHaveText(labelToDefaults.startTime.default);
+
+        await browser.waitUntil(
+            async () =>
+                (
+                    await browser.getTextByLabel(labelToDefaults.status.label)
+                ).match(/Succeeded/) !== null,
+            {
+                timeout: 20000,
+                interval: 50,
+                timeoutMsg: "Job did not reach succeeded status after 20s.",
+            }
+        );
+
+        webView.close();
+    });
     it("should run a python file as a job on a cluster", async () => {
         await startSyncIfStopped();
         await waitForSyncComplete();

packages/databricks-vscode/src/workspace-fs/WorkspaceFsWorkflowWrapper.ts

@@ -89,7 +89,7 @@ export class WorkspaceFsWorkflowWrapper {
         originalJson["cells"] = [bootstrapJson].concat(
             originalJson["cells"] ?? []
         );
-        this.createFile(
+        return this.createFile(
             getWrapperPath(remoteFilePath, [
                 "databricks",
                 "notebook",