Redirect bricks logs to a file as well (#280)

```json
{"level":"info","logger":"Bricks","message":"2022/12/05 22:46:15 [INFO] Remote file sync location: /Repos/kartik.gupta@databricks.com/Test","timestamp":1670276775608}
{"level":"info","logger":"Bricks","message":"","timestamp":1670276775608}
{"level":"info","logger":"Bricks","message":"2022/12/05 22:46:15 [INFO] Loading dogfood profile from /Users/kartik.gupta/.databrickscfg","timestamp":1670276775608}
{"level":"debug","logger":"Bricks","message":"2022/12/05 22:46:15 [DEBUG] GET /api/2.0/repos ","timestamp":1670276775608}
{"level":"debug","logger":"Bricks","message":"","timestamp":1670276775608}
{"level":"debug","logger":"Bricks","message":"2022/12/05 22:46:16 [DEBUG] 200 OK  {","timestamp":1670276776498}
{"level":"debug","logger":"Bricks","message":"  \"repos\": [","timestamp":1670276776499}
{"level":"debug","logger":"Bricks","message":"    {","timestamp":1670276776500}
{"level":"debug","logger":"Bricks","message":"      \"id\": 2600307357294504,","timestamp":1670276776500}
{"level":"debug","logger":"Bricks","message":"      \"path\": \"/Repos/kartik.gupta@databricks.com/Test\"","timestamp":1670276776500}
{"level":"debug","logger":"Bricks","message":"    }","timestamp":1670276776501}
{"level":"debug","logger":"Bricks","message":"  ]","timestamp":1670276776501}
{"level":"debug","logger":"Bricks","message":"} <- GET /api/2.0/repos?path_prefix=%2FRepos%2Fkartik.gupta%40databricks.com%2FTest","timestamp":1670276776501}
{"level":"debug","logger":"Bricks","message":"","timestamp":1670276776502}
{"level":"info","logger":"Bricks","message":"2022/12/05 22:46:17 [INFO] Initial Sync Complete","timestamp":1670276777509}
{"level":"info","logger":"Bricks","message":"","timestamp":1670276777511}

```

Author: kartikgupta-db

packages/databricks-sdk-js/src/logging/NamedLogger.ts

@@ -89,6 +89,20 @@ export class NamedLogger {
     }
 
     log(level: string, message?: string, meta?: any) {
+        if (level === "error") {
+            if (Object(meta) === meta) {
+                meta = {
+                    ...Object.getOwnPropertyNames(meta).reduce((acc, i) => {
+                        acc[i] = (meta as any)[i];
+                        return acc;
+                    }, {} as any),
+                    ...(meta as any),
+                };
+            }
+
+            meta = {error: meta};
+        }
+
         meta = defaultRedactor.sanitize(
             meta,
             this._loggerOpts?.fieldNameDenyList
@@ -109,16 +123,7 @@ export class NamedLogger {
     }
 
     error(message?: string, obj?: any) {
-        if (Object(obj) === obj) {
-            obj = {
-                ...Object.getOwnPropertyNames(obj).reduce((acc, i) => {
-                    acc[i] = (obj as any)[i];
-                    return acc;
-                }, {} as any),
-                ...(obj as any),
-            };
-        }
-        this.log(LEVELS.error, message, {error: obj});
+        this.log(LEVELS.error, message, obj);
     }
 
     withContext({

packages/databricks-vscode/src/cli/BricksSyncParser.test.ts

@@ -1,3 +1,4 @@
+import "@databricks/databricks-sdk";
 import assert from "assert";
 import {mock} from "ts-mockito";
 import {EventEmitter} from "vscode";

packages/databricks-vscode/src/cli/BricksSyncParser.ts

@@ -1,6 +1,15 @@
+import {LEVELS, NamedLogger} from "@databricks/databricks-sdk/dist/logging";
 import {EventEmitter} from "vscode";
+import {Loggers} from "../logger";
 import {SyncState} from "../sync";
 
+const bricksLogLevelToSdk = new Map<string, LEVELS>([
+    ["DEBUG", LEVELS.debug],
+    ["INFO", LEVELS.info],
+    ["WARN", LEVELS.warn],
+    ["ERROR", LEVELS.error],
+]);
+
 export class BricksSyncParser {
     private filesBeingUploaded = new Set<string>();
     private filesBeingDeleted = new Set<string>();
@@ -118,8 +127,19 @@ export class BricksSyncParser {
     // TODO: Use structed logging to compute the sync state here
     public process(data: string) {
         const logLines = data.split("\n");
+        let currentLogLevel: LEVELS = LEVELS.info;
         for (let i = 0; i < logLines.length; i++) {
             const line = logLines[i];
+            const typeMatch = line.match(
+                /[0-9]+(?:\/[0-9]+)+ [0-9]+(?::[0-9]+)+ \[(.+)\]/
+            );
+            if (typeMatch) {
+                currentLogLevel =
+                    bricksLogLevelToSdk.get(typeMatch[1]) ?? currentLogLevel;
+            }
+
+            NamedLogger.getOrCreate(Loggers.Bricks).log(currentLogLevel, line);
+
             this.parseForActionsInitiated(line);
             this.parseForUploadCompleted(line);
             this.parseForDeleteCompleted(line);

packages/databricks-vscode/src/cli/BricksTasks.test.ts

@@ -1,4 +1,4 @@
-import "@databricks/databricks-sdk/dist";
+import "@databricks/databricks-sdk";
 import * as assert from "assert";
 import {instance, mock, when} from "ts-mockito";
 import {Uri} from "vscode";

packages/databricks-vscode/src/logger/LoggerManager.test.ts

@@ -26,21 +26,25 @@ describe(__filename, function () {
         const manager = new LoggerManager(instance(mockContext));
         await manager.initLoggers();
         NamedLogger.getOrCreate(Loggers.Extension).debug("test message");
+        NamedLogger.getOrCreate(Loggers.Bricks).debug("test message");
 
         await new Promise((resolve) =>
             setTimeout(
                 resolve,
                 new Time(0.5, TimeUnits.seconds).toMillSeconds().value
             )
         );
-
-        const rawLogs = await readFile(path.join(tempDir, "logs.json"), {
-            encoding: "utf-8",
-        });
-
-        const logs = rawLogs.split("\n");
-        assert.ok(logs.length !== 0);
-        assert.ok(logs[0].includes("test message"));
+        ["sdk-and-extension-logs.json", "bricks-logs.json"].forEach(
+            async (logfile) => {
+                const rawLogs = await readFile(path.join(tempDir, logfile), {
+                    encoding: "utf-8",
+                });
+
+                const logs = rawLogs.split("\n");
+                assert.ok(logs.length !== 0);
+                assert.ok(logs[0].includes("test message"));
+            }
+        );
     });
 
     afterEach(async () => {

packages/databricks-vscode/src/logger/LoggerManager.ts

@@ -11,6 +11,20 @@ import path from "path";
 export class LoggerManager {
     constructor(readonly context: ExtensionContext) {}
 
+    private async getLogFile(prefix: string) {
+        await mkdir(this.context.logUri.fsPath, {recursive: true});
+        const logFile = path.join(
+            this.context.logUri.fsPath,
+            `${prefix}-logs.json`
+        );
+        try {
+            await access(logFile);
+            await unlink(logFile);
+        } catch (e) {}
+
+        return logFile;
+    }
+
     private getFileTransport(
         filename: string,
         extraOptions?: Omit<
@@ -27,15 +41,12 @@ export class LoggerManager {
 
     async initLoggers() {
         const outputChannel = window.createOutputChannel("Databricks Logs");
-        await mkdir(this.context.logUri.fsPath, {recursive: true});
-        const logFile = path.join(this.context.logUri.fsPath, "logs.json");
-        try {
-            await access(logFile);
-            await unlink(logFile);
-        } catch (e) {}
-
         outputChannel.clear();
 
+        const sdkAndExtensionLogfile = await this.getLogFile(
+            "sdk-and-extension"
+        );
+
         NamedLogger.getOrCreate(
             ExposedLoggers.SDK,
             {
@@ -45,7 +56,9 @@ export class LoggerManager {
                             getOutputConsoleTransport(outputChannel, {
                                 level: "debug",
                             }),
-                            this.getFileTransport(logFile, {level: "debug"}),
+                            this.getFileTransport(sdkAndExtensionLogfile, {
+                                level: "debug",
+                            }),
                         ],
                     });
                 },
@@ -65,7 +78,32 @@ export class LoggerManager {
                             getOutputConsoleTransport(outputChannel, {
                                 level: "error",
                             }),
-                            this.getFileTransport(logFile, {level: "debug"}),
+                            this.getFileTransport(sdkAndExtensionLogfile, {
+                                level: "debug",
+                            }),
+                        ],
+                    });
+                },
+            },
+            true
+        );
+
+        const bricksLogFile = await this.getLogFile("bricks");
+        /** 
+        This logger collects all the output from bricks.
+        */
+        NamedLogger.getOrCreate(
+            "Bricks",
+            {
+                factory: (name) => {
+                    return loggers.add(name, {
+                        transports: [
+                            getOutputConsoleTransport(outputChannel, {
+                                level: "error",
+                            }),
+                            this.getFileTransport(bricksLogFile, {
+                                level: "debug",
+                            }),
                         ],
                     });
                 },
@@ -79,7 +117,9 @@ export class LoggerManager {
     }
 }
 
+/* eslint-disable @typescript-eslint/naming-convention */
 export enum Loggers {
-    // eslint-disable-next-line @typescript-eslint/naming-convention
     Extension = "Extension",
+    Bricks = "Bricks",
 }
+/* eslint-enable @typescript-eslint/naming-convention */