Deploy Databricks jobs from source folder

README.md

@@ -179,6 +179,19 @@ specify for this compilation. It will override the value if set on `build.sbt`.
 > [new issue](https://github.com/microsoft/azdo-databricks/issues)
 > on GitHub if you experience any problem.
 
+### Deploy Databricks Jobs
+
+#### Parameters
+
+- **Jobs Folder Path**: a folder that contains the jobs to be deployed. For example:
+  - `$(System.DefaultWorkingDirectory)/<artifact name>/jobs`
+- **Delete Missing Jobs (optional)**: This task will remove jobs that do not exist in the folder from the workspace when this is true.
+
+#### Supported Agents
+
+- Hosted Ubuntu 1604
+- Hosted VS2017
+
 ## Release Notes
 
 Please check the [Release Notes](https://github.com/microsoft/azdo-databricks/blob/master/docs/RELEASENOTES.md)

tasks/DeployJobsTask/DeployJobsV1/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+*.js

tasks/DeployJobsTask/DeployJobsV1/index.ts

@@ -0,0 +1,150 @@
+import tl = require('azure-pipelines-task-lib/task');
+import path = require('path');
+import fs = require('fs');
+import { async } from 'q';
+import { fstat } from 'fs';
+
+async function run() {
+    try {
+        const jobsFolderPath: string = tl.getPathInput('jobsFolderPath', true);
+        var deleteMissingJobs: boolean = tl.getBoolInput('deleteMissingJobs', false);
+        if (!deleteMissingJobs) { deleteMissingJobs = false }
+
+        if (!isDirSync(jobsFolderPath)) {
+            tl.setResult(tl.TaskResult.Failed, 'The specified path for jobs folder is a file.')
+        }
+
+        var existingJobsArray = GetExistingJobs();
+        var deployableJobsArray = GetDeployableJobs(jobsFolderPath);
+        CreateMissingJobsInDatabricks(existingJobsArray, deployableJobsArray);
+        if (deleteMissingJobs) {
+            DeleteMissingJobs(existingJobsArray, deployableJobsArray);
+        }
+    }
+    catch (err) {
+        tl.setResult(tl.TaskResult.Failed, err.message);
+    }
+}
+
+class DeployableJob {
+    constructor(private path: string, public name: string) { }
+    public deploy() {
+        tl.execSync('databricks', 'jobs create --json-file ' + this.path + ' --profile AZDO');
+    }
+    public reset(existingJobId: number) {
+        tl.execSync('databricks', 'jobs reset --json-file ' + this.path + ' --job-id ' + existingJobId + ' --profile AZDO');
+    }
+}
+
+class ExistingJob {
+    constructor(public id: number, public name: string) { }
+    public remove() {
+        tl.execSync('databricks', 'jobs delete --job-id ' + this.id + ' --profile AZDO');
+    }
+}
+
+function DeleteMissingJobs(existingJobsArray: Array<ExistingJob>, deployableJobsArray: Array<DeployableJob>) {
+    var deployableJobNamesArray = deployableJobsArray.map(function (d) { return d.name; });
+    for (var i in existingJobsArray) {
+        if (deployableJobNamesArray.indexOf(existingJobsArray[i].name) === -1) {
+            existingJobsArray[i].remove();
+        }
+    }
+}
+
+function CreateMissingJobsInDatabricks(existingJobsArray: Array<ExistingJob>, deployableJobsArray: Array<DeployableJob>) {
+    var existingJobNamesArray = existingJobsArray.map(function (e) { return e.name });
+    deployableJobsArray.forEach((deployableJob, index) => {
+        var existingJob = existingJobsArray.find((existingJob) => existingJob.name === deployableJob.name);
+        if (!existingJob) {
+            deployableJob.deploy();
+        } else {
+            deployableJob.reset(existingJob.id);
+        }
+    });
+}
+
+function GetExistingJobs(): Array<ExistingJob> {
+    var result: Array<ExistingJob> = [];
+    var listResult = tl.execSync('databricks', 'jobs list --output JSON --profile AZDO');
+    if (listResult.code != 0) {
+        tl.setResult(tl.TaskResult.Failed, "Databricks Job list failed with " + listResult.stderr);
+    } else {
+        var jobsObject = JSON.parse(listResult.stdout);
+        jobsObject.jobs.forEach((job) => {
+            result.push(new ExistingJob(job.job_id, job.settings.name));
+        });
+    }
+    return result;
+}
+
+function GetDeployableJobs(jobsFolderPath: string): Array<DeployableJob> {
+    var jobFiles = GetJobFiles(jobsFolderPath);
+    var result = [];
+    for (var i in jobFiles) {
+        let jobContent = fs.readFileSync(jobFiles[i], {
+            encoding: 'utf8'
+        }).toString();
+        let jobObject = JSON.parse(jobContent);
+        result.push(new DeployableJob(jobFiles[i], jobObject.name));
+    }
+    return result;
+}
+
+function GetJobFiles(jobsFolderPath: string): Array<string> {
+    var jobsFileNameArray = fs.readdirSync(jobsFolderPath);
+    var result = [];
+    for (var jobFileNameIndex in jobsFileNameArray) {
+        result.push(path.join(jobsFolderPath, jobsFileNameArray[jobFileNameIndex]));
+    }
+    return result;
+}
+
+function isDirSync(aPath: string) {
+    try {
+        return fs.statSync(aPath).isDirectory();
+    } catch (e) {
+        if (e.code === 'ENOENT') {
+            return false;
+        } else {
+            throw e;
+        }
+    }
+}
+
+function isPython3Selected(): boolean {
+    let pythonInfo = tl.execSync("python", "-V");
+
+    if (pythonInfo.code != 0) {
+        tl.setResult(tl.TaskResult.Failed, `Failed to check python version. ${pythonInfo.stderr}`.trim())
+    }
+
+    let version: string = "";
+
+    if (pythonInfo.stderr != "") {
+        version = pythonInfo.stderr.split(' ')[1];
+    } else if (pythonInfo.stdout != "") {
+        version = pythonInfo.stdout.split(' ')[1];
+    } else {
+        tl.setResult(tl.TaskResult.Failed, `Failed to retrieve Python Version: ${pythonInfo.stderr}`);
+        return false;
+    }
+
+    if (!version.startsWith('3')) {
+        tl.setResult(tl.TaskResult.Failed, `Active Python Version: ${version}`);
+        return false;
+    }
+
+    console.log(`Version: ${version}`);
+
+    return true;
+}
+
+let python3Selected = isPython3Selected();
+
+if (python3Selected) {
+    console.log("Python3 selected. Running...");
+    run();
+} else {
+    tl.setResult(tl.TaskResult.Failed, "You must add 'Use Python Version 3.x' as the very first task for this pipeline.");
+}

tasks/DeployJobsTask/DeployJobsV1/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "deployjobs",
+  "version": "0.1.0",
+  "description": "A Pipeline task for deploying jobs from a given folder to a Databricks workspace. Jobs should be saved in the folder as JSON files",
+  "main": "index.js",
+  "scripts": {
+    "build": "tsc",
+    "test": "npm run build && mocha tests/_suite.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/microsoft/azdo-databricks.git"
+  },
+  "keywords": [
+    "azdo",
+    "devops",
+    "databricks",
+    "data"
+  ],
+  "author": "Microsoft DevLabs",
+  "license": "MIT",
+  "dependencies": {
+    "azure-pipelines-task-lib": "^2.8.0"
+  },
+  "devDependencies": {
+    "@types/mocha": "^5.2.7",
+    "@types/node": "^12.0.8",
+    "@types/q": "^1.5.2"
+  }
+}

tasks/DeployJobsTask/DeployJobsV1/task.json

@@ -0,0 +1,38 @@
+{
+    "id": "95290384-0fd3-4bcf-98e4-5586d81588de",
+    "name": "deployjobs",
+    "friendlyName": "Deploy Databricks Jobs",
+    "description": "Recursively deploys Jobs saved as json files from a given folder to a Databricks workspace. Each job that does not exist in the Databricks workspace will be created. If the job exists it will be reset.",
+    "helpMarkDown": "",
+    "category": "Deploy",
+    "author": "Microsoft DevLabs",
+    "version": {
+        "Major": 0,
+        "Minor": 1,
+        "Patch": 0
+    },
+    "instanceNameFormat": "Deploy jobs to Workspace",
+    "inputs": [
+        {
+            "name": "jobsFolderPath",
+            "type": "filePath",
+            "label": "jobs folder",
+            "defaultValue": "",
+            "required": true,
+            "helpMarkDown": "Specify the path to the folder that contains the jobs to be published to the workspace."
+        },
+        {
+            "name": "deleteMissingJobs",
+            "type": "boolean",
+            "label": "Delete Jobs from Databricks",
+            "defaultValue": "false",
+            "required": false,
+            "helpMarkDown": "If this is true, this task will remove all jobs from Databricks that are not specified in JSON files. Jobs are matched by name."
+        }
+    ],
+    "execution": {
+        "Node": {
+            "target": "index.js"
+        }
+    }
+}

tasks/DeployJobsTask/DeployJobsV1/tsconfig.json

@@ -0,0 +1,63 @@
+{
+  "compilerOptions": {
+    /* Basic Options */
+    // "incremental": true,                   /* Enable incremental compilation */
+    "target": "es6",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    // "lib": [],                             /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                       /* Allow javascript files to be compiled. */
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./",                       /* Concatenate and emit output to single file. */
+    // "outDir": "./",                        /* Redirect output structure to the directory. */
+    // "rootDir": "./",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                     /* Enable project compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+    // "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                /* Report errors on unused locals. */
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+    /* Module Resolution Options */
+    // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                       /* List of folders to include type definitions from. */
+    // "types": [],                           /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true                   /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+    // "allowUmdGlobalAccess": true,          /* Allow accessing UMD globals from modules. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+  }
+}