Skip to content

Commit ba72969

Browse files
Merge pull request #40 from JainTwinkle/config-tool
Refactor configuration utility tool; added support for latest Ray version
2 parents 335acb2 + a0112c8 commit ba72969

File tree

2 files changed

+58
-43
lines changed

2 files changed

+58
-43
lines changed

utils/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,11 @@ Use `--help` option for all available options.
2525
- [x] Parse configs and dump configs in file to be edited later according to the res
2626
- [x] If any config value is different than default value then add that config to the --system-config parameter
2727
- [x] Dump config in a yaml file named system-cm.yaml
28-
- [ ] Change hardcoded string to dynamic to dump in system-cm.yaml
28+
- [x] Change hardcoded string to dynamic to dump in system-cm.yaml
29+
- [x] Extend to more Ray versions
2930
- [ ] Update format of system config string that works correctly
3031
- [ ] Segregate internal and external options
3132
- [ ] Spread code in multiple files
32-
- [ ] Extend to more Ray versions
3333
- [ ] Add Try-catch and checks to improve robustness
3434
- [ ] Add code to also dump Ray operator and cluster yamls
3535
- [ ] Give a sample cluster Yaml to user to let user edit cluster specific configurations, e.g., cpu, num of workers

utils/ray-res-config.py

Lines changed: 56 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/usr/bin/env python3
2+
13
import sys
24
import os
35
from typing import Dict
@@ -8,7 +10,7 @@
810
from parse import *
911
from optparse import OptionParser
1012

11-
versions_list=['1.0.0', '1.1.0','1.2.0','1.3.0','1.4.0','1.4.1','1.5.0']
13+
versions_list=['1.0.0', '1.1.0','1.2.0','1.3.0','1.4.0','1.4.1','1.5.0', '1.6.0', '1.7.0', '1.8.0']
1214

1315
res_modes_list=['relaxed', 'recommended', 'strict', 'custom']
1416

@@ -49,21 +51,20 @@ def print_ray_versions_and_modes():
4951
4. custom: Define your own preference
5052
''')
5153

52-
# Generates .conf files in configs folder in the working directory
53-
# TODO: generate ray-version dir under configs directory for better file oganization
54-
def dump_conf(ray_version, res_mode, overwrite, dirname='configs/'):
54+
# Generates .conf files in configs/RAY_VERSION folder in the working directory
55+
def dump_conf(ray_version, res_mode, overwrite, dirname='configs'):
5556
# dump the default configs in a file to let others edit further
56-
file_path = dirname+"ray-"+ray_version+"-"+res_mode+".conf"
57+
file_path = dirname+"/ray-"+ray_version+"-"+res_mode+".conf"
5758
# check if the file already exist
5859
if (not os.path.exists(file_path)) or overwrite:
5960
fd = open(file_path, "w+")
6061
fd.write("# please edit value_for_this_mode to change any configuration\n")
61-
yaml.dump(Ray_conf[ray_version], fd)
62+
yaml.dump(Ray_conf[ray_version], fd, width=float('inf'))
6263

6364
# Que: is dumping in json format better or YAML?
64-
def read_conf(ray_version, res_mode, dirname='configs/'):
65+
def read_conf(ray_version, res_mode, dirname='configs'):
6566
Ray_conf_new = Ray_conf
66-
file_path = dirname+"ray-"+ray_version+"-"+res_mode+".conf"
67+
file_path = dirname+"/"+ray_version+"/ray-"+ray_version+"-"+res_mode+".conf"
6768
fd = open(file_path, 'r')
6869
fd.readline()
6970
try:
@@ -83,40 +84,59 @@ def read_conf(ray_version, res_mode, dirname='configs/'):
8384
print(exception)
8485

8586

86-
def put_conf(ray_version, conf_name, conf_type, conf_default, conf_env):
87+
def put_conf(ray_version, conf_name, conf_type, conf_default, conf_env, conf_str):
8788
# Ray_conf[version][conf_name][type/default/env]
8889
Ray_conf[ray_version][conf_name] = dict()
8990
Ray_conf[ray_version][conf_name]['type'] = conf_type
9091
Ray_conf[ray_version][conf_name]['default'] = conf_default
9192
Ray_conf[ray_version][conf_name]['value_for_this_mode'] = conf_default
9293
Ray_conf[ray_version][conf_name]['env'] = conf_env
94+
Ray_conf[ray_version][conf_name]['config_string'] = conf_str
95+
96+
# parse env. variable from config string
97+
def get_env(conf_default):
98+
conf_env = ""
99+
if 'getenv' in conf_default:
100+
_,conf_env,_ = parse('{}etenv("{}"){}', conf_default)
101+
return conf_env
102+
103+
# get the default value of the configuration
104+
def get_default(conf_default, conf_type):
105+
if 'env_' in conf_default:
106+
_, conf_default = parse('{}, {})', conf_default)
107+
elif '?' in conf_default:
108+
if_str, true_str, false_str = parse('{} ? {} : {})', conf_default)
109+
if '!=' in if_str:
110+
conf_default = false_str
111+
else:
112+
conf_default = true_str
113+
elif 'std::string' in conf_default:
114+
is_eq, conf_default = parse('{} std::string("{}"))', conf_default)
115+
# if the condition is != (not equal) then Ray expect the opposite value
116+
if is_eq[-2:] == "!=":
117+
if conf_type == "bool":
118+
conf_default = str(not int(conf_default))
119+
else:
120+
print ("Unable to parse string %s" % conf_default)
121+
sys.exit(-1)
122+
123+
return conf_default
93124

94125
def parse_ray_config(ray_version, sig_str, is_multiline):
95126
# print("In parse_ray_config: %s" % sig_str)
96127
conf_type, conf_name, conf_default = parse("RAY_CONFIG({}, {}, {}", sig_str)
97128

98129
# replace if default has comment in it
99130
conf_default = re.sub('(?:/\*(.*?)\*/)', '', conf_default)
100-
conf_env = ''
131+
conf_str = ''
101132
if is_multiline:
102-
# get the default value and associated env variable
103-
if '?' in conf_default:
104-
# TODO: make the parsing conditions more general
105-
if 'RAY_preallocate_plasma_memory' in conf_default and ray_version == '1.5.0':
106-
conf_env = 'RAY_preallocate_plasma_memory'
107-
_, conf_default = parse('{}: {})', conf_default)
108-
else:
109-
_, conf_env,_, conf_default = parse('{} ? {} : {}("{}"))', conf_default)
110-
elif 'getenv' in conf_default:
111-
_, conf_env, is_eq, _, conf_default = parse('{} getenv("{}") {} std::{}("{}"))', conf_default)
112-
if is_eq == "!=" and conf_type == "bool":
113-
conf_default = str(not int(conf_default))
114-
elif 'env_' in conf_default:
115-
_, conf_env, conf_default = parse('{}("{}", {})', conf_default)
133+
conf_str = conf_default[:-1]
134+
conf_env = get_env(conf_default)
135+
conf_default = get_default(conf_default, conf_type)
116136
conf_default = conf_default.rstrip(')')
117137
# print(conf_type, conf_name, conf_default, conf_env)
118138
# Access values like this: Ray_conf[ray_version][conf_name][type/default/env]
119-
put_conf(ray_version, conf_name, conf_type, conf_default, conf_env)
139+
put_conf(ray_version, conf_name, conf_type, conf_default, conf_env, conf_str)
120140

121141
# for multi-line signatures
122142
def is_balanced_parenthesis(str_list):
@@ -196,19 +216,19 @@ def total_config(ray_version):
196216
def fetch_configs_from_git(ray_versions, res_modes, overwrite):
197217
# get configs from file or git for each ray_version
198218
for ray_version in ray_versions:
199-
out_dir = "configs"
219+
out_dir = "configs/"+ray_version
200220
# create dir if not present
201221
if not os.path.exists(out_dir):
202222
os.makedirs(out_dir)
203-
out_filename = "%s/ray-%s-config-def.h" % (out_dir,ray_version)
223+
out_filename = "%s/ray-%s-config-def.h" % (out_dir, ray_version)
204224
# wget it from git if file not present
205225
if not os.path.exists(out_filename):
206226
url = 'https://raw.githubusercontent.com/ray-project/ray/ray-%s/src/ray/common/ray_config_def.h' % ray_version
207227
wget.download(url, out=out_filename)
208228
parse_config_file(out_filename, ray_version)
209229
total_config(ray_version)
210230
for res_mode in res_modes:
211-
dump_conf(ray_version, res_mode, overwrite)
231+
dump_conf(ray_version, res_mode, overwrite, out_dir)
212232
print_colored(OK, "All conf files saved!\nDONE!")
213233

214234
# generate config json string for system-cm yaml
@@ -226,17 +246,12 @@ def gen_system_conf(conf_list, verbose):
226246
if verbose:
227247
print("Version 1.4.0 specific configs")
228248
conf_string = get_conf_string(conf_list)
229-
# FIXME: this should not be hardcoded and can be made a dict in python to be
230-
# loaded as yaml instead of a string
231-
sys_conf = """
232-
apiVersion: v1
233-
data:
234-
system_config: '{%s}'
235-
kind: ConfigMap
236-
metadata:
237-
name: system-config-json
238-
""" % (conf_string)
239-
return yaml.load(sys_conf, yaml.Loader)
249+
conf_str = '{%s}' % (conf_string)
250+
sys_conf = {'apiVersion': 'v1',
251+
'data': {'system_config': conf_str},
252+
'kind': 'ConfigMap',
253+
'metadata': {'name' : 'system-config-json'}}
254+
return sys_conf
240255

241256
# print next steps on how to use generated system-cm.yaml
242257
# TODO: generalize next steps for different deploy stratagies
@@ -328,15 +343,15 @@ def main(argv):
328343
if opts.lists:
329344
print_ray_versions_and_modes()
330345
sys.exit(0)
331-
346+
332347
# validate ray version; print list of supported versions if input is invalid
333348
if opts.ray_version not in versions_list:
334349
print_colored(FAIL, "Ray version %s not supported!" % opts.ray_version)
335350
print_ray_versions_and_modes()
336351
sys.exit(1)
337352

338353
# validate resiliency profile/mode input (case insensitive)
339-
# print list of supported versions and modes if input is unsupported
354+
# print list of supported versions and modes if input is unsupported
340355
if opts.res_mode.lower() not in res_modes_list:
341356
print_colored(FAIL, "Resiliency profile %s not supported!" % opts.res_mode)
342357
print_ray_versions_and_modes()

0 commit comments

Comments
 (0)