Upgrade: NCJ SDK to 5.0.3 (#1917)

.style.yapf

@@ -0,0 +1,8 @@
+[style]
+based_on_style=google
+spaces_before_comment=4
+split_before_logical_operator=True
+indent_width=4
+column_limit=120
+split_arguments_when_comma_terminated=True
+blank_line_before_nested_class_or_def=False

.vsts/win/win-dependencies.yml

@@ -22,5 +22,6 @@ steps:
       node --version
       exec { npm i -g npm }
       exec { npm install }
+      exec { python -m pip install --upgrade pip }
       exec { pip install -r python/requirements.txt }
     displayName: Install dependencies

Readme.md

@@ -20,9 +20,9 @@ To build Batch Explorer yourself see this [wiki](https://github.com/Azure/BatchE
 * Request a new feature on [GitHub](https://github.com/Azure/BatchExplorer/issues)
 * Vote for popular [feature requests](https://github.com/Azure/BatchExplorer/issues?utf8=%E2%9C%93&q=is%3Aopen+is%3Aissue+label%3Afeature+sort%3Areactions-%2B1-desc+)
 * File a bug on [GitHub](https://github.com/Azure/BatchExplorer/issues)
-`
+
 ## Developers
-[Dev docs](docs/readdme.md)
+[Dev docs](docs/readme.md)
 
 ## Data/Telemetry.
 
@@ -35,4 +35,5 @@ Please go see [How to disable crash reporting and telemetry](https://github.com/
 Copyright (c) Microsoft Corporation. All rights reserved.
 
 Batch Explorer is licensed under MIT [See license](LICENSE)
+
 Some icons are under Creative Commons Attribution-ShareAlike 3.0 Unported [See license](app/assets/images/logos/LICENSE)

docs/setup.md

@@ -2,7 +2,7 @@
 
 
 ## 1. Prerequisites
-Follow the **Building Batch Explorer yourself** instruction of the [main readme](../Readme.md) in order to install all require dependencies.
+Follow the **Building Batch Explorer yourself** instruction of the [wiki](https://github.com/Azure/BatchExplorer/wiki/Build-batch-explorer-yourself) in order to install all require dependencies.
 
 **Use `npm` 5 and above. This ensures a consistent build environment with the right set of dependencies**
 

package.json

@@ -60,7 +60,7 @@
     "webpack": "node --trace-deprecation --max_old_space_size=4096 node_modules/webpack/bin/webpack.js",
     "webpack:stats": "cross-env NODE_ENV=production npm run -s webpack -- --profile --json > coverage/webpack-stats.json",
     "webpack-dev-server": "node --trace-deprecation --max_old_space_size=4096 node_modules/webpack-dev-server/bin/webpack-dev-server.js",
-    "rebuild:keytar": "electron-rebuild -w keytar -p -f",
+    "rebuild:keytar": "electron-rebuild -w keytar -f",
     "postinstall": "npm run -s rebuild:keytar"
   },
   "license": "MIT",

python/controllers/ncj_controller.py

@@ -1,7 +1,5 @@
-import azure.batch.models.batch_error
 from server.app import app
 from jsonrpc import JsonRpcRequest
-from jsonrpc.error import JsonRpcError
 
 @app.procedure("submit-ncj-job")
 def submit_ncj_job(request: JsonRpcRequest, template, parameters):

python/jsonrpc/request.py

@@ -5,7 +5,6 @@
 from .error import JsonRpcParseError
 stream_loop = asyncio.new_event_loop()
 
-
 def worker():
     stream_loop.run_forever()
     return

python/main.py

@@ -1,16 +1,17 @@
-"""
-    Main module
-"""
-
+# """
+#     Main module
+# """
 import logging
 import signal
 import sys
 import server.websocket_server
 
 signal.signal(signal.SIGINT, signal.SIG_DFL)
 
+
 def setup_logging():
-    logging.basicConfig(format='%(message)s')
+    logging.basicConfig(format='%(message)s', level="INFO")
+
 
 def run():
     """
@@ -25,5 +26,7 @@ def run():
     ws_server = server.websocket_server.WebsocketServer(port)
     ws_server.run_forever()
 
+
+
 if __name__ == "__main__":
     run()

python/requirements.txt

@@ -1,5 +1,5 @@
-websockets==3.3
-pylint==1.6.5
-azure-batch-extensions==3.1.2
-pyinstaller==3.3.1
-azure-batch==4.1.1
+websockets==7.0
+pylint==2.2.2
+azure-batch-extensions==5.0.3
+pyinstaller==3.4
+azure-batch==6.0.0

python/server/aad_auth.py

@@ -1,7 +1,6 @@
-import azext.batch as batch
+import azext.batch
 from msrestazure.azure_active_directory import AdalAuthentication
 
-
 class BatchAccount:
     def __init__(self, account_id: str, name: str, account_endpoint: str, subscription_id: str):
         self.id = account_id
@@ -29,10 +28,10 @@ def __init__(self, batchToken: str, armToken: str, armUrl: str, storage_endpoint
         self.storage_endpoint = storage_endpoint
         self.account = account
 
-        self.client = batch.BatchExtensionsClient(
+        self.client = azext.batch.BatchExtensionsClient(
             credentials=self.batchCreds,
             batch_account=self.account.name,
-            base_url='https://{0}'.format(account.account_endpoint),
+            batch_url='https://{0}'.format(account.account_endpoint),
             subscription_id=account.subscription_id,
             mgmt_credentials=self.armCreds,
             mgmt_base_url=self.armUrl,

python/server/app.py

@@ -1,10 +1,10 @@
 import inspect
 import traceback
-from jsonrpc.error import JsonRpcMethodNotFoundError, JsonRpcInvalidParamsError, JsonRpcError
-import azext.batch.errors as batch_ext_error
-import azure.batch.models.batch_error as batch_error
+import azext.batch
+import azure.batch.models as batch_models
 import azure.common
 import logging
+from jsonrpc.error import JsonRpcMethodNotFoundError, JsonRpcInvalidParamsError, JsonRpcError
 
 class BatchExplorerApp:
     """
@@ -56,10 +56,10 @@ async def call_procedure(self, request):
             except azure.common.AzureMissingResourceHttpError as e:
                 # pylint: disable=E1101
                 raise JsonRpcError(e.status_code, str(e), {})
-            except batch_error.BatchErrorException as e:
+            except batch_models.BatchErrorException as e:
                 # pylint: disable=E1101
                 raise JsonRpcError(e.response.status_code, e.message.value, e.response.json())
-            except batch_ext_error.MissingParameterValue as e:
+            except azext.batch.errors.MissingParameterValue as e:
                 raise JsonRpcInvalidParamsError(str(e), {
                     'paramName': e.parameter_name,
                     'paramDescription': e.parameter_description,

src/@batch-flask/utils/logging/node-logger.ts

@@ -13,6 +13,7 @@ stream.pipe(process.stderr);
 export interface NodeLoggerConfig {
     name: string;
     path?: string;
+    json?: boolean;
 }
 
 /**
@@ -29,21 +30,29 @@ export class NodeLogger implements Logger {
         const transports: Transport[] = [
             new winston.transports.Console({
                 format: winston.format.combine(
-                    winston.format.label({label: config.name, message: true}),
+                    winston.format.label({ label: config.name, message: true }),
                     winston.format.colorize(),
                     winston.format.simple(),
                 ),
             }),
         ];
 
         if (config.path) {
+
+            let format;
+            if (config.json) {
+                format = winston.format.combine(
+                    winston.format.timestamp(),
+                    winston.format.json(),
+                );
+            } else {
+                format = winston.format.simple();
+            }
+
             transports.push(new DailyRotateFile({
                 maxFiles: 3,
                 filename: config.path,
-                format: winston.format.combine(
-                    winston.format.timestamp(),
-                    winston.format.json(),
-                ),
+                format,
             }));
         }
         this._logger = winston.createLogger({

src/app/components/layout/footer/rpc-server-status/rpc-server-status.component.ts

@@ -18,8 +18,7 @@ import "./rpc-server-status.scss";
     changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class RpcServerStatusComponent implements OnDestroy {
-    @HostBinding("class.connected")
-    public connected = false;
+    @HostBinding("class.connected") public connected = false;
 
     public get title() {
         if (this.connected) {
@@ -46,6 +45,7 @@ export class RpcServerStatusComponent implements OnDestroy {
     public ngOnDestroy() {
         this._sub.unsubscribe();
     }
+
     public showContextMenu() {
         const items: any[] = [
             new ContextMenuItem("Restart service", () => this._restartServer()),

src/app/services/python-rpc/python-rpc.service.ts

@@ -1,13 +1,12 @@
 import { Injectable, NgZone } from "@angular/core";
 import { ServerError } from "@batch-flask/core";
-import { ElectronRemote } from "@batch-flask/electron";
 import { SecureUtils, log } from "@batch-flask/utils";
 import { ArmBatchAccount } from "app/models";
 import { JsonRpcRequest, JsonRpcResponse, RequestContainer, RequestOptions } from "app/models/python-rpc";
 import { BatchExplorerService } from "app/services/batch-explorer.service";
 import { PythonRpcServerProcess } from "client/python-process";
-import { AsyncSubject, BehaviorSubject, Observable, Subject, forkJoin } from "rxjs";
-import { catchError, first, share, switchMap, tap } from "rxjs/operators";
+import { AsyncSubject, BehaviorSubject, Observable, Subject, forkJoin, of } from "rxjs";
+import { catchError, delay, first, retryWhen, share, switchMap, take, tap } from "rxjs/operators";
 import { AdalService } from "../adal";
 import { BatchAccountService } from "../batch-account";
 
@@ -22,7 +21,6 @@ export class PythonRpcService {
     private _serverProcess: PythonRpcServerProcess;
 
     constructor(
-        remote: ElectronRemote,
         private accountService: BatchAccountService,
         private adalService: AdalService,
         private _zone: NgZone,
@@ -40,7 +38,7 @@ export class PythonRpcService {
 
     public async startServer() {
         await this._serverProcess.start();
-        this.resetConnection();
+        return this.resetConnection();
     }
 
     public stopServer() {
@@ -49,14 +47,21 @@ export class PythonRpcService {
 
     public async restartServer() {
         await this._serverProcess.restart();
-        this.resetConnection();
+        return this.resetConnection();
+    }
+
+    public resetConnection() {
+        return of(null).pipe(
+            switchMap(() => this.connect()),
+            retryWhen(errors => errors.pipe(delay(1000), take(10))),
+        ).toPromise();
     }
 
     /**
      * Connect to the rpc server using websocket.
      * Call this if the connection got cut to try again.
      */
-    public resetConnection(): Observable<any> {
+    public connect(): Observable<any> {
         this._serverProcess.port.then((port) => {
             this._ready = new AsyncSubject<any>();
             const socket = this._socket = new WebSocket(`ws://127.0.0.1:${port}/ws`);
@@ -85,7 +90,7 @@ export class PythonRpcService {
                 this._retryCount++;
                 log.info(`Websocket connection closed. Retrying to connect in ${waitingTime}s`);
                 setTimeout(() => {
-                    this.resetConnection();
+                    this.connect();
                 }, waitingTime * 1000);
             };
 

src/client/logger/loggers.ts

@@ -8,16 +8,19 @@ const logsFolder = ClientConstants.logsFolder;
 export const logger = new NodeLogger({
     name: "BatchExplorer Main",
     path: path.join(logsFolder, "client.log"),
+    json: true,
 });
 
 export const pythonLogger = new NodeLogger({
     name: "BatchExplorer Python",
     path: path.join(logsFolder, "python-server.log"),
+    json: false,
 });
 
 export const rendererLogger = new NodeLogger({
     name: "BatchExplorer Renderer",
     path: path.join(logsFolder, "app.log"),
+    json: true,
 });
 
 export function initLogger() {

src/client/python-process/python-rpc-server-process.ts

@@ -27,15 +27,17 @@ export class PythonRpcServerProcess {
             return;
         }
         log.info(`Python path is: '${data.cmd}', Args: ${data.args}`);
-        const child = this._spawedProcess = spawn(data.cmd, [...data.args]);
+        const child = this._spawedProcess = spawn(data.cmd, ["-u", ...data.args], {
+
+        });
         pythonLogger.info("========================= STARTING PYTHON RPC SERVER PROCESS =========================");
 
         child.stdout.on("data", (data) => {
             pythonLogger.info(data.toString());
         });
 
         child.stderr.on("data", (data) => {
-            pythonLogger.error(data.toString());
+            pythonLogger.info(data.toString());
         });
 
         child.on("exit", (code) => {