NNI on Windows for NNI Remote mode #1073
Merged
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
added 30 commits
April 2, 2019 10:41
QuanluZhang
reviewed
May 13, 2019
src/nni_manager/common/utils.ts
Outdated
| * Use '/' to join path instead of '\' for all kinds of platform | ||
| * @param path | ||
| */ | ||
| function pathJoin(...paths: any[]): string{ |
QuanluZhang
reviewed
May 13, 2019
| this.criticalError(new NNIError('Job metrics error', `Job metrics error: ${err.message}`, err)); | ||
| }); | ||
| }); | ||
| this.trainingService.addTrialJobMetricListener(this.trialJobMetricListener); |
QuanluZhang
reviewed
May 13, 2019
| @@ -438,7 +437,7 @@ class RemoteMachineTrainingService implements TrainingService { | |||
| */ | |||
| private getLocalGpuMetricCollectorDir(): string { | |||
| let userName: string = path.basename(os.homedir()); //get current user name of os | |||
| return `${os.tmpdir()}/${userName}/nni/scripts/`; | |||
| return path.join(os.tmpdir(), userName, 'nni', 'scripts'); | |||
QuanluZhang
reviewed
May 13, 2019
| @@ -481,7 +480,7 @@ class RemoteMachineTrainingService implements TrainingService { | |||
| private async initRemoteMachineOnConnected(rmMeta: RemoteMachineMeta, conn: Client): Promise<void> { | |||
| // Create root working directory after ssh connection is ready | |||
| await this.generateGpuMetricsCollectorScript(rmMeta.username); //generate gpu script in local machine first, will copy to remote machine later | |||
| const nniRootDir: string = `${os.tmpdir()}/nni`; | |||
| const nniRootDir: string = getRemoteTmpDir(this.remoteOS) + '/nni'; | |||
QuanluZhang
reviewed
May 13, 2019
|
|
||
| //create tmp trial working folder locally. | ||
| await cpp.exec(`cp -r ${this.trialConfig.codeDir}/* ${trialLocalTempFolder}`); | ||
| await execCopydir(this.trialConfig.codeDir+'/*',trialLocalTempFolder); |
QuanluZhang
reviewed
May 13, 2019
src/nni_manager/common/utils.ts
Outdated
| if(dir === '/'){ | ||
| dir = dir + path; | ||
| }else{ | ||
| dir = dir + '/' + path; |
demianzhang
force-pushed
the
dev-win-remote
branch
from
85303ed
to
a2b9538
Compare
chicm-ms
reviewed
May 15, 2019
| if(typeof cuda_visible_device === 'string' && cuda_visible_device.length > 0) { | ||
| command = `CUDA_VISIBLE_DEVICES=${cuda_visible_device} ${this.trialConfig.command}`; | ||
| } else { | ||
| command = `CUDA_VISIBLE_DEVICES=" " ${this.trialConfig.command}`; | ||
| command = `CUDA_VISIBLE_DEVICES='-1' ${this.trialConfig.command}`; |
There was a problem hiding this comment.
Suggest to keep the original CUDA_VISIBLE_DEVICES=" " which already tested.
chicm-ms
reviewed
May 15, 2019
test/pipelines-it-remote-windows.yml
Outdated
| displayName: 'build nni bdsit_wheel' | ||
| - task: SSH@0 | ||
| inputs: | ||
| sshEndpoint: remote_nni-ci-gpu-01 |
There was a problem hiding this comment.
this endpoint remote_nni-ci-gpu-01 should not be hard coded.
chicm-ms
reviewed
May 15, 2019
test/pipelines-it-remote-windows.yml
Outdated
| displayName: 'Start docker' | ||
| - task: DownloadSecureFile@1 | ||
| inputs: | ||
| secureFile: remote_ci_private_key |
There was a problem hiding this comment.
suggest to change this key file configurable from pipeline settings.
chicm-ms
reviewed
May 15, 2019
src/nni_manager/core/nnimanager.ts
Outdated
| @@ -76,6 +77,11 @@ class NNIManager implements Manager { | |||
| status: 'INITIALIZED', | |||
| errors: [] | |||
| }; | |||
| this.trialJobMetricListener = (metric: TrialJobMetric) => { | |||
| this.onTrialJobMetrics(metric).catch((err: Error) => { | |||
| this.criticalError(new NNIError('Job metrics error', `Job metrics error: ${err.message}`, err)); | |||
There was a problem hiding this comment.
In order to handle string type err, please reference this PR #1064 to change it like this: this.criticalError(NNIError.FromError(err, 'Job metrics error: '));
|
please update doc accordingly |
xuehui1991
reviewed
May 23, 2019
chicm-ms
approved these changes
May 23, 2019
QuanluZhang
approved these changes
May 23, 2019
xuehui1991
approved these changes
May 24, 2019
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
#1053
Install nni on Windows, run experiments in remote linux machines
Pipeline:
copy nni project to remote and build wheel in remote, install the nni wheel in docker and finish the integration test.