1
+ #!/usr/bin/env python3
2
+
1
3
import sys
2
4
import os
3
5
from typing import Dict
8
10
from parse import *
9
11
from optparse import OptionParser
10
12
11
- versions_list = ['1.0.0' , '1.1.0' ,'1.2.0' ,'1.3.0' ,'1.4.0' ,'1.4.1' ,'1.5.0' ]
13
+ versions_list = ['1.0.0' , '1.1.0' ,'1.2.0' ,'1.3.0' ,'1.4.0' ,'1.4.1' ,'1.5.0' , '1.6.0' , '1.7.0' , '1.8.0' ]
12
14
13
15
res_modes_list = ['relaxed' , 'recommended' , 'strict' , 'custom' ]
14
16
@@ -49,21 +51,20 @@ def print_ray_versions_and_modes():
49
51
4. custom: Define your own preference
50
52
''' )
51
53
52
- # Generates .conf files in configs folder in the working directory
53
- # TODO: generate ray-version dir under configs directory for better file oganization
54
- def dump_conf (ray_version , res_mode , overwrite , dirname = 'configs/' ):
54
+ # Generates .conf files in configs/RAY_VERSION folder in the working directory
55
+ def dump_conf (ray_version , res_mode , overwrite , dirname = 'configs' ):
55
56
# dump the default configs in a file to let others edit further
56
- file_path = dirname + "ray-" + ray_version + "-" + res_mode + ".conf"
57
+ file_path = dirname + "/ ray-" + ray_version + "-" + res_mode + ".conf"
57
58
# check if the file already exist
58
59
if (not os .path .exists (file_path )) or overwrite :
59
60
fd = open (file_path , "w+" )
60
61
fd .write ("# please edit value_for_this_mode to change any configuration\n " )
61
- yaml .dump (Ray_conf [ray_version ], fd )
62
+ yaml .dump (Ray_conf [ray_version ], fd , width = float ( 'inf' ) )
62
63
63
64
# Que: is dumping in json format better or YAML?
64
- def read_conf (ray_version , res_mode , dirname = 'configs/ ' ):
65
+ def read_conf (ray_version , res_mode , dirname = 'configs' ):
65
66
Ray_conf_new = Ray_conf
66
- file_path = dirname + "ray-" + ray_version + "-" + res_mode + ".conf"
67
+ file_path = dirname + "/" + ray_version + "/ ray-"+ ray_version + "-" + res_mode + ".conf"
67
68
fd = open (file_path , 'r' )
68
69
fd .readline ()
69
70
try :
@@ -83,40 +84,59 @@ def read_conf(ray_version, res_mode, dirname='configs/'):
83
84
print (exception )
84
85
85
86
86
- def put_conf (ray_version , conf_name , conf_type , conf_default , conf_env ):
87
+ def put_conf (ray_version , conf_name , conf_type , conf_default , conf_env , conf_str ):
87
88
# Ray_conf[version][conf_name][type/default/env]
88
89
Ray_conf [ray_version ][conf_name ] = dict ()
89
90
Ray_conf [ray_version ][conf_name ]['type' ] = conf_type
90
91
Ray_conf [ray_version ][conf_name ]['default' ] = conf_default
91
92
Ray_conf [ray_version ][conf_name ]['value_for_this_mode' ] = conf_default
92
93
Ray_conf [ray_version ][conf_name ]['env' ] = conf_env
94
+ Ray_conf [ray_version ][conf_name ]['config_string' ] = conf_str
95
+
96
+ # parse env. variable from config string
97
+ def get_env (conf_default ):
98
+ conf_env = ""
99
+ if 'getenv' in conf_default :
100
+ _ ,conf_env ,_ = parse ('{}etenv("{}"){}' , conf_default )
101
+ return conf_env
102
+
103
+ # get the default value of the configuration
104
+ def get_default (conf_default , conf_type ):
105
+ if 'env_' in conf_default :
106
+ _ , conf_default = parse ('{}, {})' , conf_default )
107
+ elif '?' in conf_default :
108
+ if_str , true_str , false_str = parse ('{} ? {} : {})' , conf_default )
109
+ if '!=' in if_str :
110
+ conf_default = false_str
111
+ else :
112
+ conf_default = true_str
113
+ elif 'std::string' in conf_default :
114
+ is_eq , conf_default = parse ('{} std::string("{}"))' , conf_default )
115
+ # if the condition is != (not equal) then Ray expect the opposite value
116
+ if is_eq [- 2 :] == "!=" :
117
+ if conf_type == "bool" :
118
+ conf_default = str (not int (conf_default ))
119
+ else :
120
+ print ("Unable to parse string %s" % conf_default )
121
+ sys .exit (- 1 )
122
+
123
+ return conf_default
93
124
94
125
def parse_ray_config (ray_version , sig_str , is_multiline ):
95
126
# print("In parse_ray_config: %s" % sig_str)
96
127
conf_type , conf_name , conf_default = parse ("RAY_CONFIG({}, {}, {}" , sig_str )
97
128
98
129
# replace if default has comment in it
99
130
conf_default = re .sub ('(?:/\*(.*?)\*/)' , '' , conf_default )
100
- conf_env = ''
131
+ conf_str = ''
101
132
if is_multiline :
102
- # get the default value and associated env variable
103
- if '?' in conf_default :
104
- # TODO: make the parsing conditions more general
105
- if 'RAY_preallocate_plasma_memory' in conf_default and ray_version == '1.5.0' :
106
- conf_env = 'RAY_preallocate_plasma_memory'
107
- _ , conf_default = parse ('{}: {})' , conf_default )
108
- else :
109
- _ , conf_env ,_ , conf_default = parse ('{} ? {} : {}("{}"))' , conf_default )
110
- elif 'getenv' in conf_default :
111
- _ , conf_env , is_eq , _ , conf_default = parse ('{} getenv("{}") {} std::{}("{}"))' , conf_default )
112
- if is_eq == "!=" and conf_type == "bool" :
113
- conf_default = str (not int (conf_default ))
114
- elif 'env_' in conf_default :
115
- _ , conf_env , conf_default = parse ('{}("{}", {})' , conf_default )
133
+ conf_str = conf_default [:- 1 ]
134
+ conf_env = get_env (conf_default )
135
+ conf_default = get_default (conf_default , conf_type )
116
136
conf_default = conf_default .rstrip (')' )
117
137
# print(conf_type, conf_name, conf_default, conf_env)
118
138
# Access values like this: Ray_conf[ray_version][conf_name][type/default/env]
119
- put_conf (ray_version , conf_name , conf_type , conf_default , conf_env )
139
+ put_conf (ray_version , conf_name , conf_type , conf_default , conf_env , conf_str )
120
140
121
141
# for multi-line signatures
122
142
def is_balanced_parenthesis (str_list ):
@@ -196,19 +216,19 @@ def total_config(ray_version):
196
216
def fetch_configs_from_git (ray_versions , res_modes , overwrite ):
197
217
# get configs from file or git for each ray_version
198
218
for ray_version in ray_versions :
199
- out_dir = "configs"
219
+ out_dir = "configs/" + ray_version
200
220
# create dir if not present
201
221
if not os .path .exists (out_dir ):
202
222
os .makedirs (out_dir )
203
- out_filename = "%s/ray-%s-config-def.h" % (out_dir ,ray_version )
223
+ out_filename = "%s/ray-%s-config-def.h" % (out_dir , ray_version )
204
224
# wget it from git if file not present
205
225
if not os .path .exists (out_filename ):
206
226
url = 'https://raw.githubusercontent.com/ray-project/ray/ray-%s/src/ray/common/ray_config_def.h' % ray_version
207
227
wget .download (url , out = out_filename )
208
228
parse_config_file (out_filename , ray_version )
209
229
total_config (ray_version )
210
230
for res_mode in res_modes :
211
- dump_conf (ray_version , res_mode , overwrite )
231
+ dump_conf (ray_version , res_mode , overwrite , out_dir )
212
232
print_colored (OK , "All conf files saved!\n DONE!" )
213
233
214
234
# generate config json string for system-cm yaml
@@ -226,17 +246,12 @@ def gen_system_conf(conf_list, verbose):
226
246
if verbose :
227
247
print ("Version 1.4.0 specific configs" )
228
248
conf_string = get_conf_string (conf_list )
229
- # FIXME: this should not be hardcoded and can be made a dict in python to be
230
- # loaded as yaml instead of a string
231
- sys_conf = """
232
- apiVersion: v1
233
- data:
234
- system_config: '{%s}'
235
- kind: ConfigMap
236
- metadata:
237
- name: system-config-json
238
- """ % (conf_string )
239
- return yaml .load (sys_conf , yaml .Loader )
249
+ conf_str = '{%s}' % (conf_string )
250
+ sys_conf = {'apiVersion' : 'v1' ,
251
+ 'data' : {'system_config' : conf_str },
252
+ 'kind' : 'ConfigMap' ,
253
+ 'metadata' : {'name' : 'system-config-json' }}
254
+ return sys_conf
240
255
241
256
# print next steps on how to use generated system-cm.yaml
242
257
# TODO: generalize next steps for different deploy stratagies
@@ -328,15 +343,15 @@ def main(argv):
328
343
if opts .lists :
329
344
print_ray_versions_and_modes ()
330
345
sys .exit (0 )
331
-
346
+
332
347
# validate ray version; print list of supported versions if input is invalid
333
348
if opts .ray_version not in versions_list :
334
349
print_colored (FAIL , "Ray version %s not supported!" % opts .ray_version )
335
350
print_ray_versions_and_modes ()
336
351
sys .exit (1 )
337
352
338
353
# validate resiliency profile/mode input (case insensitive)
339
- # print list of supported versions and modes if input is unsupported
354
+ # print list of supported versions and modes if input is unsupported
340
355
if opts .res_mode .lower () not in res_modes_list :
341
356
print_colored (FAIL , "Resiliency profile %s not supported!" % opts .res_mode )
342
357
print_ray_versions_and_modes ()
0 commit comments