diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000000..8faca8d1bb --- /dev/null +++ b/.style.yapf @@ -0,0 +1,8 @@ +[style] +based_on_style=google +spaces_before_comment=4 +split_before_logical_operator=True +indent_width=4 +column_limit=120 +split_arguments_when_comma_terminated=True +blank_line_before_nested_class_or_def=False diff --git a/.vsts/win/win-dependencies.yml b/.vsts/win/win-dependencies.yml index e41b7b00de..4140cecf72 100644 --- a/.vsts/win/win-dependencies.yml +++ b/.vsts/win/win-dependencies.yml @@ -22,5 +22,6 @@ steps: node --version exec { npm i -g npm } exec { npm install } + exec { python -m pip install --upgrade pip } exec { pip install -r python/requirements.txt } displayName: Install dependencies diff --git a/Readme.md b/Readme.md index 35d867f837..d35a09f29f 100644 --- a/Readme.md +++ b/Readme.md @@ -20,9 +20,9 @@ To build Batch Explorer yourself see this [wiki](https://github.com/Azure/BatchE * Request a new feature on [GitHub](https://github.com/Azure/BatchExplorer/issues) * Vote for popular [feature requests](https://github.com/Azure/BatchExplorer/issues?utf8=%E2%9C%93&q=is%3Aopen+is%3Aissue+label%3Afeature+sort%3Areactions-%2B1-desc+) * File a bug on [GitHub](https://github.com/Azure/BatchExplorer/issues) -` + ## Developers -[Dev docs](docs/readdme.md) +[Dev docs](docs/readme.md) ## Data/Telemetry. @@ -35,4 +35,5 @@ Please go see [How to disable crash reporting and telemetry](https://github.com/ Copyright (c) Microsoft Corporation. All rights reserved. Batch Explorer is licensed under MIT [See license](LICENSE) + Some icons are under Creative Commons Attribution-ShareAlike 3.0 Unported [See license](app/assets/images/logos/LICENSE) diff --git a/docs/setup.md b/docs/setup.md index 9fcea64770..5fc9a7ae51 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -2,7 +2,7 @@ ## 1. Prerequisites -Follow the **Building Batch Explorer yourself** instruction of the [main readme](../Readme.md) in order to install all require dependencies. +Follow the **Building Batch Explorer yourself** instruction of the [wiki](https://github.com/Azure/BatchExplorer/wiki/Build-batch-explorer-yourself) in order to install all require dependencies. **Use `npm` 5 and above. This ensures a consistent build environment with the right set of dependencies** diff --git a/package.json b/package.json index 1ef7dc52b0..51dbee39b4 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "webpack": "node --trace-deprecation --max_old_space_size=4096 node_modules/webpack/bin/webpack.js", "webpack:stats": "cross-env NODE_ENV=production npm run -s webpack -- --profile --json > coverage/webpack-stats.json", "webpack-dev-server": "node --trace-deprecation --max_old_space_size=4096 node_modules/webpack-dev-server/bin/webpack-dev-server.js", - "rebuild:keytar": "electron-rebuild -w keytar -p -f", + "rebuild:keytar": "electron-rebuild -w keytar -f", "postinstall": "npm run -s rebuild:keytar" }, "license": "MIT", diff --git a/python/controllers/ncj_controller.py b/python/controllers/ncj_controller.py index 33d04b8a17..8c3f6851d7 100644 --- a/python/controllers/ncj_controller.py +++ b/python/controllers/ncj_controller.py @@ -1,7 +1,5 @@ -import azure.batch.models.batch_error from server.app import app from jsonrpc import JsonRpcRequest -from jsonrpc.error import JsonRpcError @app.procedure("submit-ncj-job") def submit_ncj_job(request: JsonRpcRequest, template, parameters): diff --git a/python/jsonrpc/request.py b/python/jsonrpc/request.py index bcf04d5fe2..ec8df876fe 100644 --- a/python/jsonrpc/request.py +++ b/python/jsonrpc/request.py @@ -5,7 +5,6 @@ from .error import JsonRpcParseError stream_loop = asyncio.new_event_loop() - def worker(): stream_loop.run_forever() return diff --git a/python/main.py b/python/main.py index c0f4eb4ea4..4dc5c77066 100644 --- a/python/main.py +++ b/python/main.py @@ -1,7 +1,6 @@ -""" - Main module -""" - +# """ +# Main module +# """ import logging import signal import sys @@ -9,8 +8,10 @@ signal.signal(signal.SIGINT, signal.SIG_DFL) + def setup_logging(): - logging.basicConfig(format='%(message)s') + logging.basicConfig(format='%(message)s', level="INFO") + def run(): """ @@ -25,5 +26,7 @@ def run(): ws_server = server.websocket_server.WebsocketServer(port) ws_server.run_forever() + + if __name__ == "__main__": run() diff --git a/python/requirements.txt b/python/requirements.txt index 81677b9750..ecefb78187 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,5 +1,5 @@ -websockets==3.3 -pylint==1.6.5 -azure-batch-extensions==3.1.2 -pyinstaller==3.3.1 -azure-batch==4.1.1 +websockets==7.0 +pylint==2.2.2 +azure-batch-extensions==5.0.3 +pyinstaller==3.4 +azure-batch==6.0.0 diff --git a/python/server/aad_auth.py b/python/server/aad_auth.py index f9702a5a5b..d647a56c8a 100644 --- a/python/server/aad_auth.py +++ b/python/server/aad_auth.py @@ -1,7 +1,6 @@ -import azext.batch as batch +import azext.batch from msrestazure.azure_active_directory import AdalAuthentication - class BatchAccount: def __init__(self, account_id: str, name: str, account_endpoint: str, subscription_id: str): self.id = account_id @@ -29,10 +28,10 @@ def __init__(self, batchToken: str, armToken: str, armUrl: str, storage_endpoint self.storage_endpoint = storage_endpoint self.account = account - self.client = batch.BatchExtensionsClient( + self.client = azext.batch.BatchExtensionsClient( credentials=self.batchCreds, batch_account=self.account.name, - base_url='https://{0}'.format(account.account_endpoint), + batch_url='https://{0}'.format(account.account_endpoint), subscription_id=account.subscription_id, mgmt_credentials=self.armCreds, mgmt_base_url=self.armUrl, diff --git a/python/server/app.py b/python/server/app.py index a714c58bdb..1ab7526054 100644 --- a/python/server/app.py +++ b/python/server/app.py @@ -1,10 +1,10 @@ import inspect import traceback -from jsonrpc.error import JsonRpcMethodNotFoundError, JsonRpcInvalidParamsError, JsonRpcError -import azext.batch.errors as batch_ext_error -import azure.batch.models.batch_error as batch_error +import azext.batch +import azure.batch.models as batch_models import azure.common import logging +from jsonrpc.error import JsonRpcMethodNotFoundError, JsonRpcInvalidParamsError, JsonRpcError class BatchExplorerApp: """ @@ -56,10 +56,10 @@ async def call_procedure(self, request): except azure.common.AzureMissingResourceHttpError as e: # pylint: disable=E1101 raise JsonRpcError(e.status_code, str(e), {}) - except batch_error.BatchErrorException as e: + except batch_models.BatchErrorException as e: # pylint: disable=E1101 raise JsonRpcError(e.response.status_code, e.message.value, e.response.json()) - except batch_ext_error.MissingParameterValue as e: + except azext.batch.errors.MissingParameterValue as e: raise JsonRpcInvalidParamsError(str(e), { 'paramName': e.parameter_name, 'paramDescription': e.parameter_description, diff --git a/src/@batch-flask/utils/logging/node-logger.ts b/src/@batch-flask/utils/logging/node-logger.ts index f9f266a1b5..359526d1a8 100644 --- a/src/@batch-flask/utils/logging/node-logger.ts +++ b/src/@batch-flask/utils/logging/node-logger.ts @@ -13,6 +13,7 @@ stream.pipe(process.stderr); export interface NodeLoggerConfig { name: string; path?: string; + json?: boolean; } /** @@ -29,7 +30,7 @@ export class NodeLogger implements Logger { const transports: Transport[] = [ new winston.transports.Console({ format: winston.format.combine( - winston.format.label({label: config.name, message: true}), + winston.format.label({ label: config.name, message: true }), winston.format.colorize(), winston.format.simple(), ), @@ -37,13 +38,21 @@ export class NodeLogger implements Logger { ]; if (config.path) { + + let format; + if (config.json) { + format = winston.format.combine( + winston.format.timestamp(), + winston.format.json(), + ); + } else { + format = winston.format.simple(); + } + transports.push(new DailyRotateFile({ maxFiles: 3, filename: config.path, - format: winston.format.combine( - winston.format.timestamp(), - winston.format.json(), - ), + format, })); } this._logger = winston.createLogger({ diff --git a/src/app/components/layout/footer/rpc-server-status/rpc-server-status.component.ts b/src/app/components/layout/footer/rpc-server-status/rpc-server-status.component.ts index 0081096a53..17dbc9f500 100644 --- a/src/app/components/layout/footer/rpc-server-status/rpc-server-status.component.ts +++ b/src/app/components/layout/footer/rpc-server-status/rpc-server-status.component.ts @@ -18,8 +18,7 @@ import "./rpc-server-status.scss"; changeDetection: ChangeDetectionStrategy.OnPush, }) export class RpcServerStatusComponent implements OnDestroy { - @HostBinding("class.connected") - public connected = false; + @HostBinding("class.connected") public connected = false; public get title() { if (this.connected) { @@ -46,6 +45,7 @@ export class RpcServerStatusComponent implements OnDestroy { public ngOnDestroy() { this._sub.unsubscribe(); } + public showContextMenu() { const items: any[] = [ new ContextMenuItem("Restart service", () => this._restartServer()), diff --git a/src/app/services/python-rpc/python-rpc.service.ts b/src/app/services/python-rpc/python-rpc.service.ts index 2de2054c3d..0757a576ab 100644 --- a/src/app/services/python-rpc/python-rpc.service.ts +++ b/src/app/services/python-rpc/python-rpc.service.ts @@ -1,13 +1,12 @@ import { Injectable, NgZone } from "@angular/core"; import { ServerError } from "@batch-flask/core"; -import { ElectronRemote } from "@batch-flask/electron"; import { SecureUtils, log } from "@batch-flask/utils"; import { ArmBatchAccount } from "app/models"; import { JsonRpcRequest, JsonRpcResponse, RequestContainer, RequestOptions } from "app/models/python-rpc"; import { BatchExplorerService } from "app/services/batch-explorer.service"; import { PythonRpcServerProcess } from "client/python-process"; -import { AsyncSubject, BehaviorSubject, Observable, Subject, forkJoin } from "rxjs"; -import { catchError, first, share, switchMap, tap } from "rxjs/operators"; +import { AsyncSubject, BehaviorSubject, Observable, Subject, forkJoin, of } from "rxjs"; +import { catchError, delay, first, retryWhen, share, switchMap, take, tap } from "rxjs/operators"; import { AdalService } from "../adal"; import { BatchAccountService } from "../batch-account"; @@ -22,7 +21,6 @@ export class PythonRpcService { private _serverProcess: PythonRpcServerProcess; constructor( - remote: ElectronRemote, private accountService: BatchAccountService, private adalService: AdalService, private _zone: NgZone, @@ -40,7 +38,7 @@ export class PythonRpcService { public async startServer() { await this._serverProcess.start(); - this.resetConnection(); + return this.resetConnection(); } public stopServer() { @@ -49,14 +47,21 @@ export class PythonRpcService { public async restartServer() { await this._serverProcess.restart(); - this.resetConnection(); + return this.resetConnection(); + } + + public resetConnection() { + return of(null).pipe( + switchMap(() => this.connect()), + retryWhen(errors => errors.pipe(delay(1000), take(10))), + ).toPromise(); } /** * Connect to the rpc server using websocket. * Call this if the connection got cut to try again. */ - public resetConnection(): Observable { + public connect(): Observable { this._serverProcess.port.then((port) => { this._ready = new AsyncSubject(); const socket = this._socket = new WebSocket(`ws://127.0.0.1:${port}/ws`); @@ -85,7 +90,7 @@ export class PythonRpcService { this._retryCount++; log.info(`Websocket connection closed. Retrying to connect in ${waitingTime}s`); setTimeout(() => { - this.resetConnection(); + this.connect(); }, waitingTime * 1000); }; diff --git a/src/client/logger/loggers.ts b/src/client/logger/loggers.ts index 78be396fe8..8e257a89ce 100644 --- a/src/client/logger/loggers.ts +++ b/src/client/logger/loggers.ts @@ -8,16 +8,19 @@ const logsFolder = ClientConstants.logsFolder; export const logger = new NodeLogger({ name: "BatchExplorer Main", path: path.join(logsFolder, "client.log"), + json: true, }); export const pythonLogger = new NodeLogger({ name: "BatchExplorer Python", path: path.join(logsFolder, "python-server.log"), + json: false, }); export const rendererLogger = new NodeLogger({ name: "BatchExplorer Renderer", path: path.join(logsFolder, "app.log"), + json: true, }); export function initLogger() { diff --git a/src/client/python-process/python-rpc-server-process.ts b/src/client/python-process/python-rpc-server-process.ts index 9ebf221ac2..97dfd9a80a 100644 --- a/src/client/python-process/python-rpc-server-process.ts +++ b/src/client/python-process/python-rpc-server-process.ts @@ -27,7 +27,9 @@ export class PythonRpcServerProcess { return; } log.info(`Python path is: '${data.cmd}', Args: ${data.args}`); - const child = this._spawedProcess = spawn(data.cmd, [...data.args]); + const child = this._spawedProcess = spawn(data.cmd, ["-u", ...data.args], { + + }); pythonLogger.info("========================= STARTING PYTHON RPC SERVER PROCESS ========================="); child.stdout.on("data", (data) => { @@ -35,7 +37,7 @@ export class PythonRpcServerProcess { }); child.stderr.on("data", (data) => { - pythonLogger.error(data.toString()); + pythonLogger.info(data.toString()); }); child.on("exit", (code) => {