NNI on Windows for NNI Remote mode

README.md

@@ -106,15 +106,15 @@ We encourage researchers and students leverage these projects to accelerate the
 
 ## **Install & Verify**
 
-If you choose NNI Windows local mode and you use PowerShell to run script for the first time, you need to **run PowerShell as administrator** with this command first:
+If you are using NNI on Windows and use PowerShell to run script for the first time, you need to **run PowerShell as administrator** with this command first:
 
 ```bash
     Set-ExecutionPolicy -ExecutionPolicy Unrestricted
 ```
 
 **Install through pip** 	
 
-* We support Linux, MacOS and Windows(local mode) in current stage, Ubuntu 16.04 or higher, MacOS 10.14.1 along with Windows 10.1809 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
+* We support Linux, MacOS and Windows(local, remote and pai mode) in current stage, Ubuntu 16.04 or higher, MacOS 10.14.1 along with Windows 10.1809 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
 
 Linux and MacOS
 
@@ -131,12 +131,12 @@ python -m pip install --upgrade nni
 Note:
 
 * `--user` can be added if you want to install NNI in your home directory, which does not require any special privileges.
-* Currently NNI on Windows only support local mode. Anaconda or Miniconda is highly recommended to install NNI on Windows.
+* Currently NNI on Windows support local, remote and pai mode. Anaconda or Miniconda is highly recommended to install NNI on Windows.
 * If there is any error like `Segmentation fault`, please refer to [FAQ](docs/en_US/FAQ.md)
 
 **Install through source code**
 
-* We support Linux (Ubuntu 16.04 or higher), MacOS (10.14.1) and Windows local mode (10.1809) in our current stage. 
+* We support Linux (Ubuntu 16.04 or higher), MacOS (10.14.1) and Windows (10.1809) in our current stage. 
 
 Linux and MacOS
 
@@ -160,7 +160,7 @@ Windows
 
 For the system requirements of NNI, please refer to [Install NNI](docs/en_US/Installation.md)
 
-For NNI Windows local mode, please refer to [NNI Windows local mode](docs/en_US/WindowsLocalMode.md)
+For NNI on Windows, please refer to [NNI on Windows](docs/en_US/NniOnWindows.md)
 
 **Verify install**
 

deployment/pypi/Makefile

@@ -20,22 +20,28 @@ ifeq ($(version_ts), true)
 	NNI_VERSION_VALUE := $(NNI_VERSION_VALUE).$(TIME_STAMP)
 endif
 NNI_VERSION_TEMPLATE = 999.0.0-developing
-
+NNI_YARN_TARBALL ?= $(CWD)nni-yarn.tar.gz
+NNI_YARN_FOLDER ?= $(CWD)nni-yarn
+NNI_YARN := PATH=$(CWD)node-$(OS_SPEC)-x64/bin:$${PATH} $(NNI_YARN_FOLDER)/bin/yarn
 .PHONY: build
 build:
 	python3 -m pip install --user --upgrade setuptools wheel
-	wget https://aka.ms/nni/nodejs-download/$(OS_SPEC) -O $(CWD)node-$(OS_SPEC)-x64.tar.xz
+	wget -q https://aka.ms/nni/nodejs-download/$(OS_SPEC) -O $(CWD)node-$(OS_SPEC)-x64.tar.xz
 	rm -rf $(CWD)node-$(OS_SPEC)-x64
 	mkdir $(CWD)node-$(OS_SPEC)-x64
 	tar xf $(CWD)node-$(OS_SPEC)-x64.tar.xz -C node-$(OS_SPEC)-x64 --strip-components 1
-	cd $(CWD)../../src/nni_manager && yarn && yarn build
-	cd $(CWD)../../src/webui && yarn && yarn build
+	wget -q https://aka.ms/yarn-download -O $(NNI_YARN_TARBALL)
+	rm -rf $(NNI_YARN_FOLDER)
+	mkdir $(NNI_YARN_FOLDER)
+	tar -xf $(NNI_YARN_TARBALL) -C $(NNI_YARN_FOLDER) --strip-components 1
+	cd $(CWD)../../src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build
+	cd $(CWD)../../src/webui && $(NNI_YARN) && $(NNI_YARN) build
 	rm -rf $(CWD)nni
 	cp -r $(CWD)../../src/nni_manager/dist $(CWD)nni
 	cp -r $(CWD)../../src/webui/build $(CWD)nni/static
 	cp $(CWD)../../src/nni_manager/package.json $(CWD)nni
 	sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(CWD)nni/package.json
-	cd $(CWD)nni && yarn --prod
+	cd $(CWD)nni && $(NNI_YARN) --prod
 	cd $(CWD) && sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && python3 setup.py bdist_wheel -p $(WHEEL_SPEC)
 	cd $(CWD)
 
@@ -50,4 +56,4 @@ clean:
 	rm -rf $(CWD)dist
 	rm -rf $(CWD)nni
 	rm -rf $(CWD)nni.egg-info
-	rm -rf $(CWD)node-$(OS_SPEC)-x64
+	rm -rf $(CWD)node-$(OS_SPEC)-x64

docs/en_US/FAQ.md

@@ -36,8 +36,8 @@ Unable to open the WebUI may have the following reasons:
 * If you still can't see the WebUI after you use the server IP, you can check the proxy and the firewall of your machine. Or use the browser on the machine where you start your NNI experiment.
 * Another reason may be your experiment is failed and NNI may fail to get the experiment infomation. You can check the log of NNImanager in the following directory: ~/nni/experiment/[your_experiment_id] /log/nnimanager.log
 
-### Windows local mode problems
-Please refer to [NNI Windows local mode](WindowsLocalMode.md) 
+### NNI on Windows problems
+Please refer to [NNI on Windows](NniOnWindows.md) 
 
 ### Help us improve
 Please inquiry the problem in https://github.com/Microsoft/nni/issues to see whether there are other people already reported the problem, create a new one if there are no existing issues been created.

docs/en_US/Installation.md

@@ -1,6 +1,6 @@
 # Installation of NNI
 
-Currently we support installation on Linux, Mac and Windows(local mode).
+Currently we support installation on Linux, Mac and Windows(local, remote and pai mode).
 
 ## **Installation on Linux & Mac**
 

docs/en_US/WindowsLocalMode.md → docs/en_US/NniOnWindows.md

@@ -1,6 +1,6 @@
 # Windows Local Mode (experimental feature)
 
-Currently we only support local mode on Windows. Windows 10.1809 is well tested and recommended.
+Currently we support local, remote and pai mode on Windows. Windows 10.1809 is well tested and recommended.
 
 ## **Installation on Windows**
 

docs/en_US/QuickStart.md

@@ -2,15 +2,15 @@
 
 ## Installation
 
-We support Linux MacOS and Windows(local mode) in current stage, Ubuntu 16.04 or higher, MacOS 10.14.1 and Windows 10.1809 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
+We support Linux MacOS and Windows in current stage, Ubuntu 16.04 or higher, MacOS 10.14.1 and Windows 10.1809 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
 #### Linux and MacOS
 
 ```bash
     python3 -m pip install --upgrade nni
 ```
 
 #### Windows
-If you choose Windows local mode and use PowerShell to run script, you need run below PowerShell command as administrator at first time.
+If you are using NNI on Windows, you need run below PowerShell command as administrator at first time.
 ```bash
     Set-ExecutionPolicy -ExecutionPolicy Unrestricted
 ```
@@ -151,7 +151,7 @@ Run the **config.yml** file from your command line to start MNIST experiment.
 #### Windows   
 Run the **config_windows.yml** file from your command line to start MNIST experiment.
 
-**Note**, if you're using windows local mode, it needs to change `python3` to `python` in the config.yml file, or use the config_windows.yml file to start the experiment.
+**Note**, if you're using NNI on Windows, it needs to change `python3` to `python` in the config.yml file, or use the config_windows.yml file to start the experiment.
 
 ```bash
     nnictl create --config nni/examples/trials/mnist/config_windows.yml

docs/en_US/RemoteMachineMode.md

@@ -55,7 +55,8 @@ machineList:
     username: bob
     passwd: bob123
 ```
-
+You can use different systems to run experiments on the remote machine.
+#### Linux and MacOS
 Simply filling the `machineList` section and then run:
 
 ```bash
@@ -64,5 +65,14 @@ nnictl create --config ~/nni/examples/trials/mnist-annotation/config_remote.yml
 
 to start the experiment.
 
+#### Windows
+Simply filling the `machineList` section and then run:
+
+```bash
+nnictl create --config %userprofile%\nni\examples\trials\mnist-annotation\config_remote.yml
+```
+
+to start the experiment.
+
 ## version check
 NNI support version check feature in since version 0.6, [refer](PaiMode.md)

install.ps1

@@ -15,7 +15,7 @@ $yarnUrl = "https://yarnpkg.com/latest.tar.gz"
 $unzipNodeDir = "node-v*"
 $unzipYarnDir = "yarn-v*"
 
-$NNI_DEPENDENCY_FOLDER = "C:\tmp\$env:USERNAME"
+$NNI_DEPENDENCY_FOLDER = [System.IO.Path]::GetTempPath()+$env:USERNAME
 
 $WHICH_PYTHON = where.exe python
 if($WHICH_PYTHON -eq $null){

src/nni_manager/common/utils.ts

@@ -43,11 +43,11 @@ function getExperimentRootDir(): string {
             .getLogDir();
 }
 
-function getLogDir(): string{
+function getLogDir(): string {
     return path.join(getExperimentRootDir(), 'log');
 }
 
-function getLogLevel(): string{
+function getLogLevel(): string {
     return getExperimentStartupInfo()
     .getLogLevel();
 }
@@ -149,7 +149,7 @@ function parseArg(names: string[]): string {
     return '';
 }
 
-function encodeCmdLineArgs(args:any):any{
+function encodeCmdLineArgs(args: any): any {
     if(process.platform === 'win32'){
         return JSON.stringify(args);
     }
@@ -158,7 +158,7 @@ function encodeCmdLineArgs(args:any):any{
     }
 }
 
-function getCmdPy():string{
+function getCmdPy(): string {
     let cmd = 'python3';
     if(process.platform === 'win32'){
         cmd = 'python';
@@ -390,7 +390,7 @@ async function getVersion(): Promise<string> {
 /**
  * run command as ChildProcess
  */
-function getTunerProc(command: string, stdio: StdioOptions, newCwd: string, newEnv: any): ChildProcess{
+function getTunerProc(command: string, stdio: StdioOptions, newCwd: string, newEnv: any): ChildProcess {
     let cmd: string = command;
     let arg: string[] = [];
     let newShell: boolean = true;
@@ -411,7 +411,7 @@ function getTunerProc(command: string, stdio: StdioOptions, newCwd: string, newE
 /**
  * judge whether the process is alive
  */
-async function isAlive(pid:any): Promise<boolean>{
+async function isAlive(pid:any): Promise<boolean> {
     let deferred : Deferred<boolean> = new Deferred<boolean>();
     let alive: boolean = false;
     if(process.platform ==='win32'){
@@ -439,7 +439,7 @@ async function isAlive(pid:any): Promise<boolean>{
 /**
  * kill process 
  */
-async function killPid(pid:any): Promise<void>{
+async function killPid(pid:any): Promise<void> {
     let deferred : Deferred<void> = new Deferred<void>();
     try {
         if (process.platform === "win32") {
@@ -455,7 +455,7 @@ async function killPid(pid:any): Promise<void>{
     return deferred.promise;
 }
 
-function getNewLine(): string{
+function getNewLine(): string {
     if (process.platform === "win32") {
         return "\r\n";
     }

src/nni_manager/core/nnimanager.ts

@@ -58,7 +58,8 @@ class NNIManager implements Manager {
     private status: NNIManagerStatus;
     private waitingTrials: string[];
     private trialJobs: Map<string, TrialJobDetail>;
-
+    private trialJobMetricListener: (metric: TrialJobMetric) => void;
+
     constructor() {
         this.currSubmittedTrialNum = 0;
         this.trialConcurrencyChange = 0;
@@ -76,6 +77,11 @@ class NNIManager implements Manager {
             status: 'INITIALIZED',
             errors: []
         };
+        this.trialJobMetricListener = (metric: TrialJobMetric) => {
+            this.onTrialJobMetrics(metric).catch((err: Error) => {
+                this.criticalError(NNIError.FromError(err, 'Job metrics error: '));
+            });
+        };
     }
 
     public updateExperimentProfile(experimentProfile: ExperimentProfile, updateType: ProfileUpdateType): Promise<void> {
@@ -342,6 +348,7 @@ class NNIManager implements Manager {
         if (this.dispatcher === undefined) {
             throw new Error('Error: tuner has not been setup');
         }
+        this.trainingService.removeTrialJobMetricListener(this.trialJobMetricListener); 
         this.dispatcher.sendCommand(TERMINATE);
         let tunerAlive: boolean = true;
         // gracefully terminate tuner and assessor here, wait at most 30 seconds.
@@ -589,11 +596,7 @@ class NNIManager implements Manager {
         if (this.dispatcher === undefined) {
             throw new Error('Error: tuner or job maintainer have not been setup');
         }
-        this.trainingService.addTrialJobMetricListener((metric: TrialJobMetric) => {
-            this.onTrialJobMetrics(metric).catch((err: Error) => {
-                this.criticalError(NNIError.FromError(err, 'Job metrics error: '));
-            });
-        });
+        this.trainingService.addTrialJobMetricListener(this.trialJobMetricListener);
 
         this.dispatcher.onCommand((commandType: string, content: string) => {
             this.onTunerCommand(commandType, content).catch((err: Error) => {

src/nni_manager/training_service/common/util.ts

@@ -24,7 +24,10 @@ import { getLogger } from "common/log";
 import { countFilesRecursively } from '../../common/utils'
 import * as cpp from 'child-process-promise';
 import * as cp from 'child_process';
-import { GPU_INFO_COLLECTOR_FORMAT_LINUX, GPU_INFO_COLLECTOR_FORMAT_WINDOWS } from './gpuData'
+import * as os from 'os';
+import * as fs from 'fs';
+import { getNewLine } from '../../common/utils';
+import { GPU_INFO_COLLECTOR_FORMAT_LINUX, GPU_INFO_COLLECTOR_FORMAT_WINDOWS } from './gpuData';
 import * as path from 'path';
 import { String } from 'typescript-string-operations';
 import { file } from "../../node_modules/@types/tmp";
@@ -66,6 +69,20 @@ export async function execMkdir(directory: string): Promise<void> {
     return Promise.resolve();
 }
 
+/**
+ * copy files to the directory
+ * @param source
+ * @param destination
+ */
+export async function execCopydir(source: string, destination: string): Promise<void> {
+    if (process.platform === 'win32') {
+        await cpp.exec(`powershell.exe Copy-Item ${source} -Destination ${destination} -Recurse`);
+    } else {
+        await cpp.exec(`cp -r ${source} ${destination}`);
+    }
+    return Promise.resolve();
+}
+
 /**
  * crete a new file
  * @param filename 
@@ -91,8 +108,6 @@ export function execScript(filePath: string): cp.ChildProcess {
     }
 }
 
-
-
 /**
  * output the last line of a file
  * @param filePath 
@@ -111,9 +126,9 @@ export async function execTail(filePath: string): Promise<cpp.childProcessPromis
  * delete a directory
  * @param directory 
  */
-export async function execRemove(directory: string): Promise<void>{
+export async function execRemove(directory: string): Promise<void> {
     if (process.platform === 'win32') {
-        await cpp.exec(`powershell.exe Remove-Item ${directory}`);
+        await cpp.exec(`powershell.exe Remove-Item ${directory} -Recurse -Force`);
     } else {
         await cpp.exec(`rm -rf ${directory}`);
     }
@@ -124,7 +139,7 @@ export async function execRemove(directory: string): Promise<void>{
  * kill a process
  * @param directory 
  */
-export async function execKill(pid: string): Promise<void>{
+export async function execKill(pid: string): Promise<void> {
     if (process.platform === 'win32') {
         await cpp.exec(`cmd /c taskkill /PID ${pid} /T /F`);
     } else {
@@ -138,7 +153,7 @@ export async function execKill(pid: string): Promise<void>{
  * @param  variable
  * @returns command string  
  */
-export function setEnvironmentVariable(variable: { key: string; value: string }): string{
+export function setEnvironmentVariable(variable: { key: string; value: string }): string {
     if (process.platform === 'win32') {
         return `$env:${variable.key}="${variable.value}"`;
     }
@@ -147,6 +162,32 @@ export function setEnvironmentVariable(variable: { key: string; value: string })
     }
 }
 
+/**
+ * Compress files in directory to tar file
+ * @param  source_path
+ * @param  tar_path
+ */
+export async function tarAdd(tar_path: string, source_path: string): Promise<void> {
+    if (process.platform === 'win32') {
+        tar_path = tar_path.split('\\').join('\\\\');
+        source_path = source_path.split('\\').join('\\\\');
+        let script: string[] = [];
+        script.push(
+            `import os`,
+            `import tarfile`,
+            String.Format(`tar = tarfile.open("{0}","w:gz")\r\nfor root,dir,files in os.walk("{1}"):`, tar_path, source_path),
+            `    for file in files:`,
+            `        fullpath = os.path.join(root,file)`,
+            `        tar.add(fullpath, arcname=file)`,
+            `tar.close()`);
+        await fs.promises.writeFile(path.join(os.tmpdir(), 'tar.py'), script.join(getNewLine()), { encoding: 'utf8', mode: 0o777 });
+        const tarScript: string = path.join(os.tmpdir(), 'tar.py');
+        await cpp.exec(`python ${tarScript}`);
+    } else {
+        await cpp.exec(`tar -czf ${tar_path} -C ${source_path} .`);
+    }
+    return Promise.resolve();
+}
 
 /**
  * generate script file name