Tighten ESLint rules and apply them (#188)

Author: fjakobs

.eslintrc.json

@@ -5,14 +5,39 @@
         "ecmaVersion": 6,
         "sourceType": "module"
     },
-    "plugins": ["@typescript-eslint"],
+    "plugins": ["@typescript-eslint", "no-only-tests"],
+    "extends": [
+        "prettier",
+        "eslint:recommended",
+        "plugin:@typescript-eslint/recommended"
+    ],
     "rules": {
         "@typescript-eslint/naming-convention": "warn",
         "curly": "warn",
         "eqeqeq": "warn",
-        "no-throw-literal": "warn",
-        "semi": "off"
+        "@typescript-eslint/no-non-null-assertion": "off",
+        "@typescript-eslint/no-explicit-any": "off",
+        "no-constant-condition": ["error", {"checkLoops": false}],
+        "no-empty": "off",
+        "@typescript-eslint/no-empty-function": "off",
+        "semi": "off",
+        "no-console": "error",
+        "no-only-tests/no-only-tests": "error"
     },
-    "ignorePatterns": ["out", "dist", "**/*.d.ts"],
-    "extends": ["prettier"]
+    "overrides": [
+        {
+            "files": "**/*.test.ts",
+            "rules": {
+                "no-console": "off"
+            }
+        },
+        {
+            "files": "**/apis/*/*.ts",
+            "rules": {
+                "@typescript-eslint/no-unused-vars": "off",
+                "@typescript-eslint/no-empty-interface": "off"
+            }
+        }
+    ],
+    "ignorePatterns": ["out", "dist", "**/*.d.ts"]
 }

SCRIPT.md

@@ -0,0 +1,66 @@
+1. configure plugin
+2. delete repo
+3. STUCK!!!!
+
+full sync does not clean the cache!
+I can't fix a broken state by doing a full sync!
+
+track changes to gitignore
+
+file watcher for .databricks is not working
+
+---
+
+https://docs.databricks.com/dev-tools/ide-how-to.html
+
+We hear from a lot of customers that they want to use their favourite IDE to develop for Databricks.
+Today I want to share with you what we have been working on to make this easier.
+
+A few months ago we released a demo project together with docs on IDE best practices with Datab-00ricks.
+
+We showed how to use dbx to develop locally using and IDE and then use dbx from the command line to run code on a Databricks cluster.
+
+While this worked well, it still had a few limitations:
+
+1. It wasn't integrated with the IDE. You had to context switch between the IDE, the command line, and the Databricks UI
+2. Using dbx for the edit-run cycle is rather slow
+
+We have been working on a new feature that addresses these limitations.
+
+We are happy to announce that we have released a VSCode extension for Databricks that allows you to run on Databricks code directly from your IDE.
+
+This means that you can now develop locally using your IDE and run code directly on a Databricks cluster without having to context switch between the IDE, the command line, and the Databricks UI.
+
+Let me show you how to use it.
+
+Here I have the demo project from the previous documentation article open in VSCode.
+
+Fisrt I have configured the IDE to connect to a Databricks workspace.
+
+1. I selected an interactive cluster for running my code on
+2. I created an empty repo for syncing my code to
+   2.1 creat an empty repo
+
+Let's start the sync process. Sync runs in the background and watches the file system.
+Running code always happens on the cluster.
+
+You can see the original notebook that was the starting point for the demo project.
+We have taken this notebook and
+
+1. Converted it to a python script
+2. Factored out business logic into a separate file
+3. Added unit tests for the business logic
+
+> open notebook in VSCode
+> show raw python script
+> show refactored python script
+> show library code
+> show unit tests
+> run unit tests - still locally
+> run refactored code on cluster
+> make code change
+> which country do we filter by?
+> goto def
+> change code to NLD
+> we can also run code as workflow
+> let's run the original notebook

package.json

@@ -36,6 +36,7 @@
         "conventional-changelog-cli": "^2.2.2",
         "eslint": "^8.26.0",
         "eslint-config-prettier": "^8.5.0",
+        "eslint-plugin-no-only-tests": "^3.1.0",
         "ts-mockito": "^2.6.1",
         "typescript": "^4.8.4"
     }

packages/databricks-sdk-js/src/Redactor.ts

@@ -28,7 +28,7 @@ export class Redactor {
         }
         //make a copy of the object
         obj = Object.assign({}, obj);
-        for (let key in obj) {
+        for (const key in obj) {
             if (dropFields.includes(key)) {
                 delete obj[key];
             } else if (

packages/databricks-sdk-js/src/api-client.test.ts

@@ -3,6 +3,7 @@ import ".";
 import assert from "node:assert";
 import {ApiClient} from "./api-client";
 
+// eslint-disable-next-line @typescript-eslint/no-var-requires
 const sdkVersion = require("../package.json").version;
 
 describe(__filename, () => {
@@ -11,7 +12,7 @@ describe(__filename, () => {
     });
 
     it("should create proper user agent", () => {
-        let ua = new ApiClient("unit", "3.4.5").userAgent();
+        const ua = new ApiClient("unit", "3.4.5").userAgent();
         assert.equal(
             ua,
             `unit/3.4.5 databricks-sdk-js/${sdkVersion} nodejs/${process.version.slice(

packages/databricks-sdk-js/src/api-client.ts

@@ -4,8 +4,11 @@ import {TextDecoder} from "node:util";
 import {fromDefaultChain} from "./auth/fromChain";
 import {fetch} from "./fetch";
 import {ExposedLoggers, Utils, withLogContext} from "./logging";
-import {context, Context} from "./context";
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+import {context} from "./context";
+import {Context} from "./context";
 
+// eslint-disable-next-line @typescript-eslint/no-var-requires
 const sdkVersion = require("../package.json").version;
 
 type HttpMethod = "POST" | "GET" | "DELETE" | "PATCH" | "PUT";
@@ -63,7 +66,7 @@ export class ApiClient {
     }
 
     userAgent(): string {
-        let pairs = [
+        const pairs = [
             `${this.product}/${this.productVersion}`,
             `databricks-sdk-js/${sdkVersion}`,
             `nodejs/${process.version.slice(1)}`,
@@ -80,7 +83,7 @@ export class ApiClient {
         method: HttpMethod,
         payload?: any,
         @context context?: Context
-    ): Promise<Object> {
+    ): Promise<Record<string, unknown>> {
         const credentials = await this.credentialProvider();
         const headers = {
             "Authorization": `Bearer ${credentials.token}`,
@@ -89,10 +92,10 @@ export class ApiClient {
         };
 
         // create a copy of the URL, so that we can modify it
-        let url = new URL(credentials.host.toString());
+        const url = new URL(credentials.host.toString());
         url.pathname = path;
 
-        let options: any = {
+        const options: any = {
             method,
             headers,
             agent: this.agent,
@@ -136,8 +139,8 @@ export class ApiClient {
             );
         }
 
-        let responseBody = await response.arrayBuffer();
-        let responseText = new TextDecoder().decode(responseBody);
+        const responseBody = await response.arrayBuffer();
+        const responseText = new TextDecoder().decode(responseBody);
 
         // TODO proper error handling
         if (!response.ok) {
@@ -160,7 +163,7 @@ export class ApiClient {
         }
 
         if ("error_code" in response) {
-            let message =
+            const message =
                 response.message || `HTTP error ${response.error_code}`;
             throw logAndReturnError(
                 url,

packages/databricks-sdk-js/src/apis/clusters/api.ts

@@ -94,8 +94,8 @@ export class ClustersService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         const createClusterResponse = await this.create(createCluster, context);
 
@@ -179,8 +179,8 @@ export class ClustersService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         await this.delete(deleteCluster, context);
 
@@ -273,8 +273,8 @@ export class ClustersService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         await this.edit(editCluster, context);
 
@@ -378,8 +378,8 @@ export class ClustersService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         const clusterInfo = await this.get(getRequest, context);
 
@@ -572,8 +572,8 @@ export class ClustersService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         await this.resize(resizeCluster, context);
 
@@ -655,8 +655,8 @@ export class ClustersService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         await this.restart(restartCluster, context);
 
@@ -765,8 +765,8 @@ export class ClustersService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         await this.start(startCluster, context);
 

packages/databricks-sdk-js/src/apis/commands/api.ts

@@ -62,8 +62,8 @@ export class CommandExecutionService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         await this.cancel(cancelCommand, context);
 
@@ -188,8 +188,8 @@ export class CommandExecutionService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         const created = await this.create(createContext, context);
 
@@ -296,8 +296,8 @@ export class CommandExecutionService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         const created = await this.execute(command, context);
 

packages/databricks-sdk-js/src/apis/commands/commands.integ.ts

@@ -15,15 +15,15 @@ describe(__filename, function () {
     });
 
     it("should execute python with low level API", async () => {
-        let commandsApi = new CommandExecutionService(integSetup.client);
+        const commandsApi = new CommandExecutionService(integSetup.client);
 
-        let context = await commandsApi.createAndWait({
+        const context = await commandsApi.createAndWait({
             clusterId: integSetup.cluster.id,
             language: "python",
         });
         //console.log("Execution context", context);
 
-        let status = await commandsApi.executeAndWait({
+        const status = await commandsApi.executeAndWait({
             clusterId: integSetup.cluster.id,
             contextId: context.id,
             language: "python",

packages/databricks-sdk-js/src/apis/jobs/api.ts

@@ -83,8 +83,8 @@ export class JobsService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         await this.cancelRun(cancelRun, context);
 
@@ -261,8 +261,8 @@ export class JobsService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         const run = await this.getRun(getRunRequest, context);
 
@@ -412,8 +412,8 @@ export class JobsService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         await this.repairRun(repairRun, context);
 
@@ -515,8 +515,8 @@ export class JobsService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         const runNowResponse = await this.runNow(runNow, context);
 
@@ -601,8 +601,8 @@ export class JobsService {
         options = options || {};
         options.onProgress =
             options.onProgress || (async (newPollResponse) => {});
-        let {timeout, onProgress} = options;
-        let cancellationToken = context?.cancellationToken;
+        const {timeout, onProgress} = options;
+        const cancellationToken = context?.cancellationToken;
 
         const submitRunResponse = await this.submit(submitRun, context);