Bundle watcher (#892)

## Changes
* Autogenerate BundleSchema.d.ts file (from `databricks bundle schema`
command) to provide typing for the parsed bundle object.
* Add a watcher for all DABs configuration files
* Add BundleFileSet object, to handle operations on the full bundle file
tree.
* Used these to add autocompletion for all files included in the root
bundle.yaml.

**NOTE:** This PR currently parses yamls in code. We want to move that
to databricks CLI. The implementation can is modular and can be swapped
with the CLI calls once we have those features in the CLI.

## Tests
* Unit tests
* Manual

Author: kartikgupta-db

packages/databricks-vscode/package.json

@@ -671,6 +671,7 @@
         "package:cli:link": "rm -f ./bin/databricks && mkdir -p bin && ln -s ../../../../cli/cli bin/databricks",
         "package:wrappers:write": "ts-node ./scripts/writeIpynbWrapper.ts -s ./resources/python/notebook.workflow-wrapper.py -o ./resources/python/generated/notebook.workflow-wrapper.json",
         "package:jupyter-init-script:write": "ts-node ./scripts/writeJupyterInitFileWithVersion.ts",
+        "package:bundle-schema:write": "yarn package:cli:fetch && ts-node ./scripts/writeBundleSchema.ts ./bin/databricks ./src/bundle/BundleSchema.d.ts",
         "package:compile": "yarn run esbuild:base",
         "package:copy-webview-toolkit": "cp ./node_modules/@vscode/webview-ui-toolkit/dist/toolkit.js ./out/toolkit.js",
         "esbuild:base": "esbuild ./src/extension.ts --bundle --outfile=out/extension.js --external:vscode --format=cjs --platform=node --sourcemap --target=es2019",
@@ -692,6 +693,7 @@
     "dependencies": {
         "@databricks/databricks-sdk": "file:../../vendor/databricks-sdk.tgz",
         "@databricks/databricks-vscode-types": "workspace:^",
+        "@types/lodash": "^4.14.199",
         "@vscode/debugadapter": "^1.61.0",
         "@vscode/extension-telemetry": "^0.8.1",
         "@vscode/webview-ui-toolkit": "^1.2.2",
@@ -708,7 +710,7 @@
         "@types/bcryptjs": "^2.4.2",
         "@types/chai": "^4.3.5",
         "@types/fs-extra": "^11.0.1",
-        "@types/mocha": "^10.0.1",
+        "@types/mocha": "^10.0.2",
         "@types/mock-require": "^2.0.1",
         "@types/node": "^20.4.2",
         "@types/sinonjs__fake-timers": "^8.1.2",
@@ -728,7 +730,8 @@
         "esbuild": "^0.19.4",
         "eslint": "^8.51.0",
         "fs-extra": "^11.1.1",
-        "glob": "^10.3.3",
+        "glob": "^10.3.10",
+        "json-schema-to-typescript": "^13.1.1",
         "mocha": "^10.2.0",
         "mock-require": "^3.0.3",
         "nyc": "^15.1.0",

packages/databricks-vscode/scripts/writeBundleSchema.ts

@@ -0,0 +1,25 @@
+/**
+ * This script generates the BundleSchema.d.ts file from the bundle schema.
+ * It MUST be run after a yarn package:cli:fetch
+ */
+
+import * as cp from "child_process";
+import * as fs from "fs";
+import {compileFromFile} from "json-schema-to-typescript";
+import {tmpdir} from "os";
+import path from "path";
+import {argv} from "process";
+
+const output = cp.execFileSync(argv[2], ["bundle", "schema"]);
+
+const tmpFile = path.join(tmpdir(), "BundleSchema.json");
+fs.writeFileSync(tmpFile, output);
+
+// eslint-disable-next-line no-console
+console.log("Bundle schema written to", tmpFile);
+
+// compile from file
+compileFromFile(tmpFile).then((ts) => fs.writeFileSync(argv[3], ts));
+
+// eslint-disable-next-line no-console
+console.log("BundleSchema.d.ts written to", argv[3]);

packages/databricks-vscode/src/bundle/BundleFileSet.test.ts

@@ -0,0 +1,195 @@
+import {Uri} from "vscode";
+import {BundleFileSet} from "./BundleFileSet";
+import {expect} from "chai";
+import path from "path";
+import * as tmp from "tmp-promise";
+import * as fs from "fs/promises";
+import {BundleSchema} from "./BundleSchema";
+import * as yaml from "yaml";
+
+describe(__filename, async function () {
+    let tmpdir: tmp.DirectoryResult;
+
+    beforeEach(async () => {
+        tmpdir = await tmp.dir({unsafeCleanup: true});
+    });
+
+    afterEach(async () => {
+        await tmpdir.cleanup();
+    });
+
+    it("should return the correct absolute path", () => {
+        const tmpdirUri = Uri.file(tmpdir.path);
+
+        const bundleFileSet = new BundleFileSet(tmpdirUri);
+
+        expect(bundleFileSet.getAbsolutePath("test.txt").fsPath).to.equal(
+            path.join(tmpdirUri.fsPath, "test.txt")
+        );
+
+        expect(
+            bundleFileSet.getAbsolutePath(Uri.file("test.txt")).fsPath
+        ).to.equal(path.join(tmpdirUri.fsPath, "test.txt"));
+    });
+
+    it("should find the correct root bundle yaml", async () => {
+        const tmpdirUri = Uri.file(tmpdir.path);
+        const bundleFileSet = new BundleFileSet(tmpdirUri);
+
+        expect(await bundleFileSet.getRootFile()).to.be.undefined;
+
+        await fs.writeFile(path.join(tmpdirUri.fsPath, "bundle.yaml"), "");
+
+        expect((await bundleFileSet.getRootFile())?.fsPath).to.equal(
+            path.join(tmpdirUri.fsPath, "bundle.yaml")
+        );
+    });
+
+    it("should return undefined if more than one root bundle yaml is found", async () => {
+        const tmpdirUri = Uri.file(tmpdir.path);
+        const bundleFileSet = new BundleFileSet(tmpdirUri);
+
+        await fs.writeFile(path.join(tmpdirUri.fsPath, "bundle.yaml"), "");
+        await fs.writeFile(path.join(tmpdirUri.fsPath, "databricks.yaml"), "");
+
+        expect(await bundleFileSet.getRootFile()).to.be.undefined;
+    });
+
+    describe("file listing", async () => {
+        beforeEach(async () => {
+            const rootBundleData: BundleSchema = {
+                include: [
+                    "included.yaml",
+                    path.join("includes", "**", "*.yaml"),
+                ],
+            };
+
+            await fs.writeFile(
+                path.join(tmpdir.path, "bundle.yaml"),
+                yaml.stringify(rootBundleData)
+            );
+
+            await fs.writeFile(path.join(tmpdir.path, "included.yaml"), "");
+            await fs.writeFile(path.join(tmpdir.path, "notIncluded.yaml"), "");
+            await fs.mkdir(path.join(tmpdir.path, "includes"));
+            await fs.writeFile(
+                path.join(tmpdir.path, "includes", "included.yaml"),
+                ""
+            );
+        });
+
+        it("should return correct included files", async () => {
+            const tmpdirUri = Uri.file(tmpdir.path);
+            const bundleFileSet = new BundleFileSet(tmpdirUri);
+
+            expect(await bundleFileSet.getIncludedFilesGlob()).to.equal(
+                `{included.yaml,${path.join("includes", "**", "*.yaml")}}`
+            );
+
+            const actual = (await bundleFileSet.getIncludedFiles())?.map(
+                (v) => v.fsPath
+            );
+            const expected = [
+                Uri.file(path.join(tmpdirUri.fsPath, "included.yaml")),
+                Uri.file(
+                    path.join(tmpdirUri.fsPath, "includes", "included.yaml")
+                ),
+            ].map((v) => v.fsPath);
+            expect(actual).to.deep.equal(expected);
+        });
+
+        it("should return all bundle files", async () => {
+            const tmpdirUri = Uri.file(tmpdir.path);
+            const bundleFileSet = new BundleFileSet(tmpdirUri);
+
+            const actual = (await bundleFileSet.allFiles()).map(
+                (v) => v.fsPath
+            );
+            const expected = [
+                Uri.joinPath(tmpdirUri, "bundle.yaml"),
+                Uri.joinPath(tmpdirUri, "included.yaml"),
+                Uri.joinPath(tmpdirUri, "includes", "included.yaml"),
+            ].map((v) => v.fsPath);
+            expect(actual).to.deep.equal(expected);
+        });
+
+        it("isRootBundleFile should return true only for root bundle file", async () => {
+            const tmpdirUri = Uri.file(tmpdir.path);
+            const bundleFileSet = new BundleFileSet(tmpdirUri);
+
+            const possibleRoots = [
+                "bundle.yaml",
+                "bundle.yml",
+                "databricks.yaml",
+                "databricks.yml",
+            ];
+
+            for (const root of possibleRoots) {
+                expect(
+                    bundleFileSet.isRootBundleFile(
+                        Uri.file(path.join(tmpdirUri.fsPath, root))
+                    )
+                ).to.be.true;
+            }
+
+            expect(
+                bundleFileSet.isRootBundleFile(
+                    Uri.file(path.join(tmpdirUri.fsPath, "bundle-wrong.yaml"))
+                )
+            ).to.be.false;
+        });
+
+        it("isIncludedBundleFile should return true only for included files", async () => {
+            const tmpdirUri = Uri.file(tmpdir.path);
+            const bundleFileSet = new BundleFileSet(tmpdirUri);
+
+            expect(
+                await bundleFileSet.isIncludedBundleFile(
+                    Uri.file(path.join(tmpdirUri.fsPath, "included.yaml"))
+                )
+            ).to.be.true;
+
+            expect(
+                await bundleFileSet.isIncludedBundleFile(
+                    Uri.file(
+                        path.join(tmpdirUri.fsPath, "includes", "included.yaml")
+                    )
+                )
+            ).to.be.true;
+
+            expect(
+                await bundleFileSet.isIncludedBundleFile(
+                    Uri.file(path.join(tmpdirUri.fsPath, "notIncluded.yaml"))
+                )
+            ).to.be.false;
+        });
+
+        it("isBundleFile should return true only for bundle files", async () => {
+            const tmpdirUri = Uri.file(tmpdir.path);
+            const bundleFileSet = new BundleFileSet(tmpdirUri);
+
+            const possibleBundleFiles = [
+                "bundle.yaml",
+                "bundle.yml",
+                "databricks.yaml",
+                "databricks.yml",
+                "included.yaml",
+                path.join("includes", "included.yaml"),
+            ];
+
+            for (const bundleFile of possibleBundleFiles) {
+                expect(
+                    await bundleFileSet.isBundleFile(
+                        Uri.file(path.join(tmpdirUri.fsPath, bundleFile))
+                    )
+                ).to.be.true;
+            }
+
+            expect(
+                await bundleFileSet.isBundleFile(
+                    Uri.file(path.join(tmpdirUri.fsPath, "notIncluded.yaml"))
+                )
+            ).to.be.false;
+        });
+    });
+});

packages/databricks-vscode/src/bundle/BundleFileSet.ts

@@ -0,0 +1,133 @@
+import {Uri} from "vscode";
+import * as glob from "glob";
+import {merge} from "lodash";
+import * as yaml from "yaml";
+import path from "path";
+import {BundleSchema} from "./BundleSchema";
+import {readFile} from "fs/promises";
+import {CachedValue} from "../locking/CachedValue";
+import minimatch from "minimatch";
+
+export async function parseBundleYaml(file: Uri) {
+    const data = yaml.parse(await readFile(file.fsPath, "utf-8"));
+    return data as BundleSchema;
+}
+
+function toGlobPath(path: string) {
+    if (process.platform === "win32") {
+        return path.replace(/\\/g, "/");
+    }
+    return path;
+}
+export class BundleFileSet {
+    private rootFilePattern: string = "{bundle,databricks}.{yaml,yml}";
+    private _mergedBundle: CachedValue<BundleSchema> =
+        new CachedValue<BundleSchema>(async () => {
+            let bundle = {};
+            await this.forEach(async (data) => {
+                bundle = merge(bundle, data);
+            });
+            return bundle as BundleSchema;
+        });
+
+    constructor(private readonly workspaceRoot: Uri) {}
+
+    getAbsolutePath(path: string | Uri) {
+        if (typeof path === "string") {
+            return Uri.joinPath(this.workspaceRoot, path);
+        }
+        return Uri.joinPath(this.workspaceRoot, path.fsPath);
+    }
+
+    async getRootFile() {
+        const rootFile = await glob.glob(
+            toGlobPath(this.getAbsolutePath(this.rootFilePattern).fsPath),
+            {nocase: process.platform === "win32"}
+        );
+        if (rootFile.length !== 1) {
+            return undefined;
+        }
+        return Uri.file(rootFile[0]);
+    }
+
+    async getIncludedFilesGlob() {
+        const rootFile = await this.getRootFile();
+        if (rootFile === undefined) {
+            return undefined;
+        }
+        const bundle = await parseBundleYaml(Uri.file(rootFile.fsPath));
+        const includedFilesGlob =
+            bundle?.include === undefined || bundle?.include.length === 0
+                ? undefined
+                : `{${bundle.include?.join(",")}}`;
+
+        return includedFilesGlob;
+    }
+
+    async getIncludedFiles() {
+        const includedFilesGlob = await this.getIncludedFilesGlob();
+        if (includedFilesGlob !== undefined) {
+            return (
+                await glob.glob(
+                    toGlobPath(
+                        path.join(this.workspaceRoot.fsPath, includedFilesGlob)
+                    ),
+                    {nocase: process.platform === "win32"}
+                )
+            ).map((i) => Uri.file(i));
+        }
+    }
+
+    async allFiles() {
+        const rootFile = await this.getRootFile();
+        if (rootFile === undefined) {
+            return [];
+        }
+
+        return [rootFile, ...((await this.getIncludedFiles()) ?? [])];
+    }
+
+    async findFileWithPredicate(predicate: (file: Uri) => Promise<boolean>) {
+        const matchedFiles: Uri[] = [];
+        for (const file of await this.allFiles()) {
+            if (await predicate(file)) {
+                matchedFiles.push(file);
+            }
+        }
+        return matchedFiles;
+    }
+
+    async forEach(f: (data: BundleSchema, file: Uri) => Promise<void>) {
+        for (const file of await this.allFiles()) {
+            await f(await parseBundleYaml(file), file);
+        }
+    }
+
+    isRootBundleFile(e: Uri) {
+        return minimatch(
+            e.fsPath,
+            toGlobPath(this.getAbsolutePath(this.rootFilePattern).fsPath)
+        );
+    }
+
+    async isIncludedBundleFile(e: Uri) {
+        let includedFilesGlob = await this.getIncludedFilesGlob();
+        if (includedFilesGlob === undefined) {
+            return false;
+        }
+        includedFilesGlob = this.getAbsolutePath(includedFilesGlob).fsPath;
+        return minimatch(e.fsPath, toGlobPath(includedFilesGlob));
+    }
+
+    async isBundleFile(e: Uri) {
+        return this.isRootBundleFile(e) || (await this.isIncludedBundleFile(e));
+    }
+
+    async invalidateMergedBundleCache() {
+        await this._mergedBundle.invalidate();
+    }
+
+    get mergedBundle() {
+        return this._mergedBundle.value;
+    }
+}