Skip to content

Commit b3aedff

Browse files
larponlarpon
authored
toml: convert yaml value checks in iarna test suite (#12629)
1 parent 6f297cd commit b3aedff

File tree

2 files changed

+114
-17
lines changed

2 files changed

+114
-17
lines changed

‎.github/workflows/toml_ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ on:
1010

1111
jobs:
1212
toml-module-pass-external-test-suites:
13-
runs-on: ubuntu-20.04
13+
runs-on: ubuntu-18.04
1414
timeout-minutes: 10
1515
env:
1616
TOML_BS_TESTS_PATH: vlib/toml/tests/testdata/burntsushi/toml-test

‎vlib/toml/tests/iarna.toml-spec-tests_test.v

Lines changed: 113 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,27 @@ const (
1111
// Can be set to `true` to skip tests that stress test the parser
1212
// by having large data amounts - these pass - but slow down the test run
1313
skip_large_files = false
14+
// Can be set to `true` to skip tests that triggers a slow conversion
15+
// process that uses `python` to convert from YAML to JSON.
16+
skip_yaml_conversion = false
1417

1518
// Kept for easier handling of future updates to the tests
1619
valid_exceptions = []string{}
1720
invalid_exceptions = []string{}
1821

19-
valid_value_exceptions = []string{}
22+
valid_value_exceptions = [
23+
'values/spec-string-basic.toml',
24+
]
25+
26+
yaml_value_exceptions = [
27+
'values/spec-float-5.toml', // YAML: "1e6", V: 1000000
28+
'values/spec-float-9.toml', // YAML: "-0e0", V: 0
29+
'values/spec-float-6.toml', // YAML: "-2E-2", V: -0.02
30+
'values/spec-float-4.toml', // YAML: "5e+22", V: 50000000000000004000000
31+
]
2032

2133
jq = os.find_abs_path_of_executable('jq') or { '' }
34+
python = os.find_abs_path_of_executable('python') or { '' }
2235
compare_work_dir_root = os.join_path(os.temp_dir(), 'v', 'toml', 'iarna')
2336
// From: https://stackoverflow.com/a/38266731/1904615
2437
jq_normalize = r'# Apply f to composite entities recursively using keys[], and to atoms
@@ -98,25 +111,71 @@ fn test_iarna_toml_spec_tests() {
98111
$if windows {
99112
relative = relative.replace('/', '\\')
100113
}
101-
if !os.exists(valid_test_file.all_before_last('.') + '.json') {
102-
println('N/A [${i + 1}/$valid_test_files.len] "$valid_test_file"...')
103-
continue
104-
}
114+
105115
// Skip the file if we know it can't be parsed or we know that the value retrieval needs work.
106116
if relative !in valid_exceptions && relative !in valid_value_exceptions {
117+
valid_test_file_name := os.file_name(valid_test_file).all_before_last('.')
118+
uses_json_format := os.exists(valid_test_file.all_before_last('.') + '.json')
119+
120+
// Use python to convert the YAML files to json - it yields some inconsistencies
121+
// so we skip some of them
122+
mut converted_from_yaml := false
123+
mut converted_json_path := ''
124+
if !uses_json_format {
125+
$if windows {
126+
println('N/A [${i + 1}/$valid_test_files.len] "$valid_test_file"...')
127+
continue
128+
}
129+
if python == '' {
130+
println('N/A [${i + 1}/$valid_test_files.len] "$valid_test_file"...')
131+
continue
132+
}
133+
if skip_yaml_conversion || relative in yaml_value_exceptions
134+
|| valid_test_file.contains('qa-') {
135+
e++
136+
println('SKIP [${i + 1}/$valid_test_files.len] "$valid_test_file" EXCEPTION [$e/$valid_value_exceptions.len]...')
137+
continue
138+
}
139+
140+
iarna_yaml_path := valid_test_file.all_before_last('.') + '.yaml'
141+
if os.exists(iarna_yaml_path) {
142+
// python -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin), sys.stdout, indent=4)' < file.yaml > file.json
143+
144+
converted_json_path = os.join_path(compare_work_dir_root,
145+
valid_test_file_name + '.yaml.json')
146+
147+
run([python, '-c',
148+
"'import sys, yaml, json; json.dump(yaml.load(sys.stdin), sys.stdout, indent=4)'",
149+
'<', iarna_yaml_path, '>', converted_json_path]) or {
150+
contents := os.read_file(iarna_yaml_path) or { panic(err) }
151+
// NOTE there's known errors with the python convertion method.
152+
// For now we just ignore them as it's a broken tool - not a wrong test-case.
153+
// Uncomment this print to see/check them.
154+
// eprintln(err.msg + '\n$contents')
155+
e++
156+
println('ERR [${i + 1}/$valid_test_files.len] "$valid_test_file" EXCEPTION [$e/$valid_value_exceptions.len]...')
157+
continue
158+
}
159+
converted_from_yaml = true
160+
}
161+
}
162+
107163
println('OK [${i + 1}/$valid_test_files.len] "$valid_test_file"...')
108164
toml_doc := toml.parse_file(valid_test_file) or { panic(err) }
109165

110-
v_toml_json_path := os.join_path(compare_work_dir_root,
111-
os.file_name(valid_test_file).all_before_last('.') + '.v.json')
166+
v_toml_json_path := os.join_path(compare_work_dir_root, valid_test_file_name +
167+
'.v.json')
112168
iarna_toml_json_path := os.join_path(compare_work_dir_root,
113-
os.file_name(valid_test_file).all_before_last('.') + '.json')
169+
valid_test_file_name + '.json')
114170

115-
os.write_file(v_toml_json_path, to_iarna(toml_doc.ast.table)) or { panic(err) }
116-
117-
iarna_json := os.read_file(valid_test_file.all_before_last('.') + '.json') or {
171+
os.write_file(v_toml_json_path, to_iarna(toml_doc.ast.table, converted_from_yaml)) or {
118172
panic(err)
119173
}
174+
175+
if converted_json_path == '' {
176+
converted_json_path = valid_test_file.all_before_last('.') + '.json'
177+
}
178+
iarna_json := os.read_file(converted_json_path) or { panic(err) }
120179
os.write_file(iarna_toml_json_path, iarna_json) or { panic(err) }
121180

122181
v_normalized_json := run([jq, '-S', '-f "$jq_normalize_path"', v_toml_json_path]) or {
@@ -197,10 +256,13 @@ fn to_iarna_time(time_str string) string {
197256
}
198257

199258
// to_iarna returns a iarna compatible json string converted from the `value` ast.Value.
200-
fn to_iarna(value ast.Value) string {
259+
fn to_iarna(value ast.Value, skip_value_map bool) string {
201260
match value {
202261
ast.Quoted {
203262
json_text := json2.Any(value.text).json_str()
263+
if skip_value_map {
264+
return '"$json_text"'
265+
}
204266
return '{ "type": "string", "value": "$json_text" }'
205267
}
206268
ast.DateTime {
@@ -218,49 +280,84 @@ fn to_iarna(value ast.Value) string {
218280
// date-time values are represented in detail. For now we follow the BurntSushi format
219281
// that expands to 6 digits which is also a valid RFC 3339 representation.
220282
json_text = to_iarna_time(json_text)
283+
if skip_value_map {
284+
return '"$json_text"'
285+
}
221286
return '{ "type": "$typ", "value": "$json_text" }'
222287
}
223288
ast.Date {
224289
json_text := json2.Any(value.text).json_str()
290+
if skip_value_map {
291+
return '"$json_text"'
292+
}
225293
return '{ "type": "date", "value": "$json_text" }'
226294
}
227295
ast.Time {
228296
mut json_text := json2.Any(value.text).json_str()
229297
json_text = to_iarna_time(json_text)
298+
if skip_value_map {
299+
return '"$json_text"'
300+
}
230301
return '{ "type": "time", "value": "$json_text" }'
231302
}
232303
ast.Bool {
233304
json_text := json2.Any(value.text.bool()).json_str()
305+
if skip_value_map {
306+
return '$json_text'
307+
}
234308
return '{ "type": "bool", "value": "$json_text" }'
235309
}
236310
ast.Null {
237311
json_text := json2.Any(value.text).json_str()
312+
if skip_value_map {
313+
return '$json_text'
314+
}
238315
return '{ "type": "null", "value": "$json_text" }'
239316
}
240317
ast.Number {
241-
if value.text.contains('inf') || value.text.contains('nan') {
242-
return '{ "type": "float", "value": "$value.text" }'
318+
if value.text.contains('inf') {
319+
mut json_text := value.text.replace('inf', '1.7976931348623157e+308') // Inconsistency ???
320+
if skip_value_map {
321+
return '$json_text'
322+
}
323+
return '{ "type": "float", "value": "$json_text" }'
324+
}
325+
if value.text.contains('nan') {
326+
mut json_text := 'null'
327+
if skip_value_map {
328+
return '$json_text'
329+
}
330+
return '{ "type": "float", "value": "$json_text" }'
243331
}
244332
if !value.text.starts_with('0x')
245333
&& (value.text.contains('.') || value.text.to_lower().contains('e')) {
246334
mut val := '$value.f64()'.replace('.e+', '.0e') // json notation
247335
if !val.contains('.') && val != '0' { // json notation
248336
val += '.0'
249337
}
338+
if skip_value_map {
339+
return '$val'
340+
}
250341
return '{ "type": "float", "value": "$val" }'
251342
}
252343
v := value.i64()
253344
// TODO workaround https://github.com/vlang/v/issues/9507
254345
if v == i64(-9223372036854775807 - 1) {
346+
if skip_value_map {
347+
return '-9223372036854775808'
348+
}
255349
return '{ "type": "integer", "value": "-9223372036854775808" }'
256350
}
351+
if skip_value_map {
352+
return '$v'
353+
}
257354
return '{ "type": "integer", "value": "$v" }'
258355
}
259356
map[string]ast.Value {
260357
mut str := '{ '
261358
for key, val in value {
262359
json_key := json2.Any(key).json_str()
263-
str += ' "$json_key": ${to_iarna(val)},'
360+
str += ' "$json_key": ${to_iarna(val, skip_value_map)},'
264361
}
265362
str = str.trim_right(',')
266363
str += ' }'
@@ -269,7 +366,7 @@ fn to_iarna(value ast.Value) string {
269366
[]ast.Value {
270367
mut str := '[ '
271368
for val in value {
272-
str += ' ${to_iarna(val)},'
369+
str += ' ${to_iarna(val, skip_value_map)},'
273370
}
274371
str = str.trim_right(',')
275372
str += ' ]\n'

0 commit comments

Comments
 (0)