Skip to content
This repository has been archived by the owner on Sep 19, 2023. It is now read-only.

WIP: Dev/support multi modes feature #140

Open
wants to merge 44 commits into
base: dev/model_studio_lite
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
2ea7334
dev/support_multi_modes_feature: add filter mode, add method Post in …
phuongnh06 Feb 26, 2021
e8cd46b
dev/support_multi_modes_feature: change response task log, add reques…
phuongnh06 Mar 3, 2021
15123b9
dev/support_multi_modes_feature: revert response get task log
phuongnh06 Mar 3, 2021
9e5ece2
dev/support_multi_modes_feature: redirect if user does not login
phuongnh06 Mar 5, 2021
6682a0e
dev/support_multi_modes_feature: remove default required_mode
phuongnh06 Mar 6, 2021
e9e687f
dev/support_multi_modes_feature: remove get config data in json
phuongnh06 Mar 9, 2021
55a5cdf
dev/support_multi_modes_feature: remove check session in filter_mode
phuongnh06 Mar 9, 2021
5427d24
Merge branch 'dev/model_studio_lite' into dev/support_multi_modes_fea…
phuongnh06 Mar 16, 2021
8ce69d6
dev/support_multi_modes_feature: fix default ngpu when release task
phuongnh06 Mar 17, 2021
849ae30
refs #56253: handle train restricted folder permission
phuongnh06 Mar 23, 2021
2d6d49c
dev/56253_handler_train_restricted_folder_permission: change function…
phuongnh06 Mar 24, 2021
53f1443
Merge pull request #143 from OpenNMT/dev/56253_handler_train_restrict…
nguyendc-systran Mar 24, 2021
ef2f765
refs #56292: auto partition corpus on new model training
Mar 25, 2021
45ce023
refs #56292: only allow testing_percent to be an int > 0
Mar 26, 2021
be3fafb
refs #55454: Add argument entity_owner
phuongnh06 Mar 29, 2021
58a269b
Merge pull request #145 from OpenNMT/dev/55454_add_argument_entity_ow…
nguyendc-systran Mar 29, 2021
9f88590
Merge pull request #146 from OpenNMT/dev/56292_partition_in_storage
nguyendc-systran Apr 1, 2021
0d13833
refs #56292: [*] use tmp file to partition call
nguyendc-systran Apr 2, 2021
48df630
refs #54931: Fix tag format
pj-liard Apr 8, 2021
8ac6979
refs #56045: allow GPU trans in as_release mode
Apr 13, 2021
f15a744
Merge pull request #149 from hieubz/dev/56045_allow_GPU_trans_in_as_r…
nguyendc-systran Apr 13, 2021
a04ba9a
refs #56551: Fix client sample size
pj-liard Apr 13, 2021
68a9aa7
refs #54931: Add tag option in snw client to launch training
pj-liard Apr 14, 2021
71af679
refs #54931: Change snw tags option : --tag -> --tags
pj-liard Apr 15, 2021
476625c
refs #54931: Add comment for snw
pj-liard Apr 15, 2021
8f99e36
refs #54931: Code refactoring for tags option
pj-liard Apr 15, 2021
41add4f
refs #54931: Return 400 status when tag is invalid
pj-liard Apr 15, 2021
1cc0312
refs #56433: Allow evaluation with Google model in mode lite
Apr 15, 2021
86c6e05
Merge pull request #151 from trungkienbkhn/dev/56433_allow_evaluation…
nguyendc-systran Apr 16, 2021
a5ce3e4
refs #54931: Fix parse tag function
pj-liard Apr 16, 2021
dd6d951
Merge pull request #148 from OpenNMT/dev/54931_possibility_to_add_tag…
nguyendc-systran Apr 16, 2021
e375d7e
Merge pull request #150 from OpenNMT/dev/56551_fix_client_sample_size…
nguyendc-systran Apr 16, 2021
05cb87b
refs #56646: add parent_task option to snw launch
Apr 28, 2021
0e44ee2
refs #56678: ngpu is 1 for train or 0 otherwise
Apr 30, 2021
55a15b2
Merge pull request #154 from OpenNMT/dev/56678_fix_ngpus_on_launcher
nguyendc-systran Apr 30, 2021
c39761b
refs #56365: change parent_task to "dependency"
May 3, 2021
35cd407
Merge pull request #153 from OpenNMT/dev/56646_add_parent_task_option…
nguyendc-systran May 3, 2021
f3a82ca
refs #56678: 0 gpus by default for score
May 4, 2021
dfa8563
refs #56381: Fix token expire after 1 day between: the end of preproc…
trungkienbkhn May 10, 2021
8f9260c
Merge pull request #156 from trungkienbkhn/dev/56381_fix_token_expire…
nguyendc-systran May 11, 2021
633e53b
refs #56678: 1 gpu to train 0 otherwise
May 11, 2021
ef77c24
refs #56757: remove build object from parent config
pj-liard May 11, 2021
4a5c0bc
Merge pull request #157 from OpenNMT/dev/56757_remove_build_object_fr…
nguyendc-systran May 11, 2021
19476ae
Merge pull request #155 from OpenNMT/dev/56678_fix_ngpus_on_launcher
nguyendc-systran May 11, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 23 additions & 3 deletions client/launcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ def confirm(prompt=None, resp=False):
"help": 'options selected to run the service'}],
['-r', '--resource',
{"help": "alternatively to `options`, resource name to use"}],
['-g', '--gpus', {"type": int, "default": 1, "help": 'number of gpus'}],
['-g', '--gpus', {"type": int, "help": 'number of gpus'}],
['-c', '--cpus', {"type": int,
"help": 'number of cpus - if not provided, will be obtained '
'from pool config'}],
Expand Down Expand Up @@ -187,6 +187,8 @@ def confirm(prompt=None, resp=False):
for arg in exec_arguments:
parser_exec.add_argument(*arg[:-1], **arg[-1])
parser_exec.add_argument('docker_command', type=str, nargs='*', help='Docker command')
parser_exec.add_argument('-e', '--entity_owner', help='entity owner')
parser_exec.add_argument('--dependency', help='dependency task (will launch after this task is terminated)')

parser_launch = subparsers.add_parser('launch',
help='launch a task on the service associated'
Expand All @@ -208,8 +210,7 @@ def confirm(prompt=None, resp=False):
help='options selected to run the service')
parser_launch.add_argument('-r', '--resource',
help="alternatively to `options`, resource name to use")
parser_launch.add_argument('-g', '--gpus', type=int, default=1,
help='number of gpus')
parser_launch.add_argument('-g', '--gpus', type=int, help='number of gpus')
parser_launch.add_argument('-c', '--cpus', type=int, help='number of cpus - if not provided, '
'will be obtained from pool config')
parser_launch.add_argument('-w', '--wait_after_launch', default=2, type=int,
Expand All @@ -236,6 +237,8 @@ def confirm(prompt=None, resp=False):
parser_launch.add_argument('--notransasrelease', action='store_true',
help='don\'t run translate as release (image >= 1.8.0)')
parser_launch.add_argument('docker_command', type=str, nargs='*', help='Docker command')
parser_launch.add_argument('--tags', nargs='*', help='model tags')
parser_launch.add_argument('--dependency', help='dependency task (will launch after this task is terminated)')

parser_list_tasks = subparsers_tasks.add_parser('list',
help='{lt} list tasks matching prefix pattern')
Expand Down Expand Up @@ -465,6 +468,12 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None):
if args.service not in service_list:
raise ValueError("ERROR: service '%s' not defined" % args.service)

if args.gpus is None:
if "train" in args.docker_command:
args.gpus = 1
else:
args.gpus = 0

if args.gpus < 0:
raise ValueError("ERROR: ngpus must be >= 0")

Expand Down Expand Up @@ -493,6 +502,8 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None):
content["name"] = args.name
if args.priority:
content["priority"] = args.priority
if args.dependency:
content["dependency"] = args.dependency

if cmd == "task" and subcmd == "launch":
if args.iterations:
Expand All @@ -516,6 +527,15 @@ def process_request(service_list, cmd, subcmd, is_json, args, auth=None):
content["totuminer"] = [(_parse_local_filename(i, files),
o) for (i, o) in ARGS.totuminer]

if args.tags:
content['tags'] = []
for tag in args.tags:
tmp = {'tag': tag}
if ':' in tag:
entity, tag_name = tag.split(':')
tmp = {'entity': entity, 'tag': tag_name}
content['tags'].append(tmp)

LOGGER.debug("sending request: %s", json.dumps(content))

launch_url = os.path.join(args.url, "task/launch", args.service)
Expand Down
Loading