-
Notifications
You must be signed in to change notification settings - Fork 35
/
errparse.jl
121 lines (111 loc) · 3.68 KB
/
errparse.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
# This deletes arbitrary tokens from files and checks that we can still parse them
# and that iteration functions are still correctly ordered.
@testset "invalid jl file parsing" begin
trav(x, f = x->nothing) = (f(x);for a in x trav(a, f) end)
trav1(x, f = x->nothing) = (f(x);if x.args !== nothing;for a in x.args trav1(a, f) end;end)
function check_err_parse(s, n = 1000)
check_str(s) # parsing works?
check_itr_order(s) # iteration produces same text?
ts = collect(tokenize(s))[1:end-1]
for i in 1:n
length(ts) == 1 && return
deleteat!(ts, rand(1:length(ts)))
check_str(untokenize(ts))
end
end
function check_str(s)
x = try
CSTParser.parse(s, true)
catch e
@info "Couldn't parse:"
@info s
rethrow(e)
end
try
trav(x)
catch e
@info "Couldn't traverse:"
@info s
rethrow(e)
end
end
function get_segs(x)
offset = 0
segs = []
for i = 1:length(x)
a = x[i]
push!(segs, offset .+ (1:a.fullspan))
offset += a.fullspan
end
segs
end
function check_itr_order(s, x = CSTParser.parse(s, true))
length(x) == 0 && return
segs = get_segs(x)
s0 = join(String(codeunits(s)[seg]) for seg in segs)
if s0 == s
for i = 1:length(x)
if length(x[i]) > 0
seg = segs[i]
s2 = String(codeunits(s)[seg])
check_itr_order(s2, x[i])
end
end
else
@info "check_itr_order failed: "
@info s
error()
end
end
comp(x, y) = x == y
function comp(x::CSTParser.EXPR, y::CSTParser.EXPR)
comp(x.head, y.head) &&
x.span == y.span &&
x.fullspan == y.fullspan &&
x.val == y.val &&
length(x) == length(y) &&
all(comp(x[i], y[i]) for i = 1:length(x))
end
function check_reparse(s0, n = 1000)
for i in 1:n
x0 = CSTParser.parse(s0, true)
ts = collect(tokenize(s0))[1:end-1]
length(ts) < 2 && return
deleteat!(ts, rand(1:length(ts)))
s1 = untokenize(ts)
length(ts) < 3 || isempty(s1) && return
x1 = CSTParser.parse(s1, true)
x2 = try
CSTParser.minimal_reparse(s0, s1, x0, x1)
catch err
@info "minimal reparse failed with"
@info "s0:"
@info codeunits(s0)
@info "s1:"
@info codeunits(s1)
rethrow(err)
end
@test comp(x1, x2) ? true : (@info(string("Comparison failed between s0:\n", s0, "\n\n and s1: \n", s1)); false)
@test join(String(codeunits(s0)[seg]) for seg in get_segs(x0)) == s0
@test join(String(codeunits(s1)[seg]) for seg in get_segs(x1)) == s1
@test join(String(codeunits(s1)[seg]) for seg in get_segs(x2)) == s1
s0 = s1
end
end
function check_dir(dir, check)
for (root, dirs, files) in walkdir(dir)
for f in files
f = joinpath(root, f)
(!isfile(f) || !endswith(f, ".jl")) && continue
@info "checking $(nameof(check)) against $f"
s = String(read(f))
if isvalid(s) && length(s) >0
check(s)
end
end
end
true
end
@test check_dir("..", check_err_parse)
@test check_dir("..", check_reparse)
end