Bundle integ merge main (#1009)

Merge main into build-integ

---------

Co-authored-by: Kartik Gupta <88345179+kartikgupta-db@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: releasebot <noreply@github.com>
Co-authored-by: kartikgupta-db <kartik.gupta@databricks.com>
Co-authored-by: eng-dev-ecosystem-bot <eng-dev-ecosystem-bot@users.noreply.github.com>
Co-authored-by: eng-dev-ecosystem-bot <110475461+eng-dev-ecosystem-bot@users.noreply.github.com>
Co-authored-by: Serge Smertin <259697+nfx@users.noreply.github.com>

Author: 6 people

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@databricks/databricks-vscode",
-    "version": "1.2.4",
+    "version": "1.2.5",
     "private": true,
     "workspaces": [
         "packages/*"

packages/databricks-vscode-types/CHANGELOG.md

@@ -1,3 +1,11 @@
+# Release: v1.2.5
+
+## packages/databricks-vscode-types
+
+## <small>1.2.5 (2024-01-10)</small>
+
+-   Update the license (#981) ([66cf3c3](https://github.com/databricks/databricks-vscode/commit/66cf3c3)), closes [#981](https://github.com/databricks/databricks-vscode/issues/981)
+
 # Release: v1.2.4
 
 ## packages/databricks-vscode-types

packages/databricks-vscode-types/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@databricks/databricks-vscode-types",
-    "version": "1.2.4",
+    "version": "1.2.5",
     "description": "Package with types and interfaces to develop extensions to the Databricks VSCode plugin",
     "main": "index.js",
     "types": "index.d.ts",

packages/databricks-vscode/CHANGELOG.md

@@ -1,3 +1,17 @@
+# Release: v1.2.5
+
+## packages/databricks-vscode
+
+## <small>1.2.5 (2024-01-10)</small>
+
+-   Explicitly clear terminal environment variables before injecting new ones.
+-   Fix race conditions around setting environment variable for the metadata url.
+-   Show warning when the extension doesn't have permissions to execute the CLI.
+-   Store `lastInstalledExtensionVersion` in the global storage so we only show `what's new` once.
+-   Update the license.
+-   Update Databricks CLI to v0.211.0.
+-   Fix the telemetry.
+
 # Release: v1.2.4
 
 ## packages/databricks-vscode

packages/databricks-vscode/package.json

@@ -4,7 +4,7 @@
     "description": "IDE support for Databricks",
     "publisher": "databricks",
     "license": "LicenseRef-LICENSE",
-    "version": "1.2.4",
+    "version": "1.2.5",
     "engines": {
         "vscode": "^1.83.0"
     },
@@ -677,7 +677,7 @@
         "useYarn": false
     },
     "cli": {
-        "version": "0.210.3"
+        "version": "0.212.0"
     },
     "scripts": {
         "vscode:prepublish": "rm -rf out && yarn run package:compile && yarn run package:wrappers:write && yarn run package:jupyter-init-script:write && yarn run package:copy-webview-toolkit && yarn run generate-telemetry",

packages/databricks-vscode/src/configuration/ConnectionManager.ts

@@ -15,6 +15,7 @@ import {ConfigModel} from "./models/ConfigModel";
 import {onError} from "../utils/onErrorDecorator";
 import {AuthProvider, ProfileAuthProvider} from "./auth/AuthProvider";
 import {Mutex} from "../locking";
+import {MetadataService} from "./auth/MetadataService";
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 const {NamedLogger} = logging;
@@ -35,6 +36,7 @@ export class ConnectionManager implements Disposable {
     private _syncDestinationMapper?: SyncDestinationMapper;
     private _clusterManager?: ClusterManager;
     private _databricksWorkspace?: DatabricksWorkspace;
+    private _metadataService: MetadataService;
 
     private readonly onDidChangeStateEmitter: EventEmitter<ConnectionState> =
         new EventEmitter();
@@ -50,7 +52,17 @@ export class ConnectionManager implements Disposable {
     public readonly onDidChangeSyncDestination =
         this.onDidChangeSyncDestinationEmitter.event;
 
-    public metadataServiceUrl?: string;
+    constructor(
+        private cli: CliWrapper,
+        private readonly configModel: ConfigModel,
+        private readonly workspaceUri: Uri,
+        private readonly customWhenContext: CustomWhenContext
+    ) {
+        this._metadataService = new MetadataService(
+            undefined,
+            NamedLogger.getOrCreate("Extension")
+        );
+    }
 
     @onError({
         popup: {prefix: "Error attaching sync destination: "},
@@ -104,12 +116,9 @@ export class ConnectionManager implements Disposable {
         this.onDidChangeClusterEmitter.fire(this.cluster);
     }
 
-    constructor(
-        private cli: CliWrapper,
-        private readonly configModel: ConfigModel,
-        private readonly workspaceUri: Uri,
-        private readonly customWhenContext: CustomWhenContext
-    ) {}
+    get metadataServiceUrl() {
+        return this._metadataService.url;
+    }
 
     public async init() {
         await this.configModel.init();
@@ -345,6 +354,11 @@ export class ConnectionManager implements Disposable {
         });
     }
 
+    async startMetadataService() {
+        await this._metadataService.listen();
+        return this._metadataService;
+    }
+
     async waitForConnect(): Promise<void> {
         if (this._state !== "CONNECTED") {
             return await new Promise((resolve) => {

packages/databricks-vscode/src/configuration/auth/MetadataServiceManager.ts

@@ -34,7 +34,6 @@ import {
 import {CustomWhenContext} from "./vscode-objs/CustomWhenContext";
 import {StateStorage} from "./vscode-objs/StateStorage";
 import path from "node:path";
-import {MetadataServiceManager} from "./configuration/auth/MetadataServiceManager";
 import {FeatureId, FeatureManager} from "./feature-manager/FeatureManager";
 import {DbConnectAccessVerifier} from "./language/DbConnectAccessVerifier";
 import {MsPythonExtensionWrapper} from "./language/MsPythonExtensionWrapper";
@@ -100,9 +99,10 @@ export async function activate(
 
     // Add the databricks binary to the PATH environment variable in terminals
     context.environmentVariableCollection.clear();
-    context.environmentVariableCollection.append(
+    context.environmentVariableCollection.persistent = false;
+    context.environmentVariableCollection.prepend(
         "PATH",
-        `${path.delimiter}${context.asAbsolutePath("./bin")}`
+        `${context.asAbsolutePath("./bin")}${path.delimiter}`
     );
 
     const loggerManager = new LoggerManager(context);
@@ -222,10 +222,9 @@ export async function activate(
             }
         })
     );
-    const metadataServiceManager = new MetadataServiceManager(
-        connectionManager
-    );
-    await metadataServiceManager.listen();
+
+    const metadataService = await connectionManager.startMetadataService();
+    context.subscriptions.push(metadataService);
 
     const workspaceFsDataProvider = new WorkspaceFsDataProvider(
         connectionManager
@@ -237,7 +236,6 @@ export async function activate(
     );
 
     context.subscriptions.push(
-        metadataServiceManager,
         window.registerTreeDataProvider(
             "workspaceFsView",
             workspaceFsDataProvider

packages/databricks-vscode/src/extension.ts

@@ -18,6 +18,7 @@ import {Context, context} from "@databricks/databricks-sdk/dist/context";
 import {DbConnectStatusBarButton} from "../language/DbConnectStatusBarButton";
 import {EnvVarGenerators, FileUtils} from "../utils";
 import {NotebookInitScriptManager} from "../language/notebooks/NotebookInitScriptManager";
+import {Mutex} from "../locking/Mutex";
 
 function isValidUserEnvPath(
     path: string | undefined,
@@ -27,7 +28,7 @@ function isValidUserEnvPath(
 }
 export class DatabricksEnvFileManager implements Disposable {
     private disposables: Disposable[] = [];
-
+    private mutex = new Mutex();
     private readonly unresolvedDatabricksEnvFile: string;
     private readonly databricksEnvPath: Uri;
     private readonly unresolvedUserEnvFile: string;
@@ -97,38 +98,31 @@ export class DatabricksEnvFileManager implements Disposable {
         this.disposables.push(
             userEnvFileWatcher,
             userEnvFileWatcher.onDidChange(async () => {
-                await this.emitToTerminal();
                 await this.writeFile();
             }, this),
             userEnvFileWatcher.onDidDelete(async () => {
-                await this.emitToTerminal();
                 await this.writeFile();
             }, this),
             userEnvFileWatcher.onDidCreate(async () => {
-                await this.emitToTerminal();
                 await this.writeFile();
             }, this),
             this.featureManager.onDidChangeState(
                 "notebooks.dbconnect",
                 async () => {
-                    await this.emitToTerminal();
                     await this.writeFile();
                 }
             ),
             this.featureManager.onDidChangeState(
                 "debugging.dbconnect",
                 () => {
-                    this.emitToTerminal();
                     this.writeFile();
                 },
                 this
             ),
             this.connectionManager.onDidChangeCluster(async () => {
-                this.emitToTerminal();
                 this.writeFile();
             }, this),
             this.connectionManager.onDidChangeState(async () => {
-                this.emitToTerminal();
                 this.writeFile();
             }, this)
         );
@@ -153,44 +147,51 @@ export class DatabricksEnvFileManager implements Disposable {
     async writeFile(@context ctx?: Context) {
         await this.connectionManager.waitForConnect();
 
-        const data = Object.entries({
-            ...(this.getDatabrickseEnvVars() || {}),
-            ...((await EnvVarGenerators.getDbConnectEnvVars(
-                this.connectionManager,
-                this.workspacePath
-            )) || {}),
-            ...this.getIdeEnvVars(),
-            ...((await this.getUserEnvVars()) || {}),
-        })
-            .filter(([, value]) => value !== undefined)
-            .map(([key, value]) => `${key}=${value}`);
-        data.sort();
+        await this.mutex.wait();
         try {
-            const oldData = await readFile(
-                this.databricksEnvPath.fsPath,
-                "utf-8"
-            );
-            if (oldData !== data.join(os.EOL)) {
-                this.onDidChangeEnvironmentVariablesEmitter.fire();
+            const data = Object.entries({
+                ...(this.getDatabrickseEnvVars() || {}),
+                ...((await EnvVarGenerators.getDbConnectEnvVars(
+                    this.connectionManager,
+                    this.workspacePath
+                )) || {}),
+                ...this.getIdeEnvVars(),
+                ...((await this.getUserEnvVars()) || {}),
+            })
+                .filter(([, value]) => value !== undefined)
+                .map(([key, value]) => `${key}=${value}`);
+            data.sort();
+            try {
+                const oldData = await readFile(
+                    this.databricksEnvPath.fsPath,
+                    "utf-8"
+                );
+                if (oldData === data.join(os.EOL)) {
+                    return;
+                }
+            } catch (e) {
+                ctx?.logger?.info("Error reading old databricks.env file", e);
             }
-        } catch (e) {
-            ctx?.logger?.info("Error reading old databricks.env file", e);
-        }
-        try {
-            await writeFile(
-                this.databricksEnvPath.fsPath,
-                data.join(os.EOL),
-                "utf-8"
-            );
-            this.dbConnectStatusBarButton.update();
-        } catch (e) {
-            ctx?.logger?.info("Error writing databricks.env file", e);
+            this.onDidChangeEnvironmentVariablesEmitter.fire();
+            try {
+                await writeFile(
+                    this.databricksEnvPath.fsPath,
+                    data.join(os.EOL),
+                    "utf-8"
+                );
+                this.dbConnectStatusBarButton.update();
+                await this.emitToTerminal();
+            } catch (e) {
+                ctx?.logger?.info("Error writing databricks.env file", e);
+            }
+        } finally {
+            this.mutex.signal();
         }
     }
 
     async emitToTerminal() {
-        await this.connectionManager.waitForConnect();
-
+        this.clearTerminalEnv();
+        this.extensionContext.environmentVariableCollection.persistent = false;
         Object.entries({
             ...(this.getDatabrickseEnvVars() || {}),
             ...this.getIdeEnvVars(),
@@ -207,6 +208,10 @@ export class DatabricksEnvFileManager implements Disposable {
                 value
             );
         });
+        this.extensionContext.environmentVariableCollection.prepend(
+            "PATH",
+            `${this.extensionContext.asAbsolutePath("./bin")}${path.delimiter}`
+        );
     }
 
     async clearTerminalEnv() {