-
Notifications
You must be signed in to change notification settings - Fork 112
/
tests.jl
309 lines (282 loc) · 10.3 KB
/
tests.jl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
function _timed_include(str::String, mod::Module=Main; use_ctime::Bool=VERSION >= v"1.9.0")
if use_ctime
compile_elapsedtimes = Base.cumulative_compile_time_ns()
end
stats = @timed Base.include(identity, mod, str)
fullpath = abspath(joinpath(Base.source_dir(), str))
# skip files which just include other files and ignore
# files outside of the oscar folder
if startswith(fullpath, Oscar.oscardir)
path = relpath(fullpath, Oscar.oscardir)
if use_ctime
compile_elapsedtimes = Base.cumulative_compile_time_ns() .- compile_elapsedtimes
compile_elapsedtimes = compile_elapsedtimes ./ 10^9
end
rtime=NaN
if use_ctime
comptime = first(compile_elapsedtimes)
rcomptime = last(compile_elapsedtimes)
println("-> Testing $path took: runtime $(round(stats.time-comptime; digits=3)) seconds + compilation $(round(comptime-rcomptime; digits=3)) seconds + recompilation $(round(rcomptime; digits=3)) seconds, $(Base.format_bytes(stats.bytes))")
return (path=>(time=stats.time-comptime, ctime=comptime-rcomptime, rctime=rcomptime, alloc=stats.bytes/2^20))
else
println("-> Testing $path took: $(round(stats.time; digits=3)) seconds, $(Base.format_bytes(stats.bytes))")
return (path=>(time=stats.time, alloc=stats.bytes/2^20))
end
else
return ()
end
end
function _gather_tests(path::AbstractString; ignore=[])
# default ignore patterns
ignorepatterns = Regex[
# this can only run on the main process and not on distributed workers
# so it is included directly in runtests
r"Serialization/IPC(\.jl)?$",
# ignore book example code (except for main file)
r"(^|/)book/.*/.*\.jl$",
]
for i in ignore
if i isa Regex
push!(ignorepatterns, i)
elseif i isa AbstractString
if endswith(i, ".jl")
push!(ignorepatterns, Regex("$i\$"))
else
push!(ignorepatterns, Regex("$i(\\.jl)?\$"))
end
else
throw(ArgumentError("invalid ignore pattern $i"))
end
end
if any(p->contains(path, p), ignorepatterns)
@info "ignore: $(relpath(path, Oscar.oscardir))"
return String[]
end
if !isabspath(path)
path = joinpath(Oscar.oscardir, path)
end
isfile(path) && return [path]
isfile("$path.jl") && return ["$path.jl"]
# if there is a runtests.jl we ignore everything else in that folder
# except for the main Oscar test dir
isfile(joinpath(path, "runtests.jl")) &&
path != joinpath(Oscar.oscardir, "test") &&
return [joinpath(path, "runtests.jl")]
tests = String[]
for entry in readdir(path; join=true)
if any(s->contains(relpath(entry, Oscar.oscardir), s), ignorepatterns)
@info "ignore: $(relpath(entry, Oscar.oscardir))"
continue
end
endswith(entry, "setup_tests.jl") && continue
# this is only for the main test/runtests.jl
endswith(entry, "runtests.jl") && continue
if isdir(entry)
append!(tests, _gather_tests(entry; ignore=ignore))
elseif isfile(entry) && endswith(entry, ".jl")
push!(tests, entry)
end
end
return tests
end
_setactiveproject(s::String) = @static if VERSION >= v"1.8"
Base.set_active_project(s)
else
Base.ACTIVE_PROJECT[] = s
end
@doc raw"""
Oscar.@_AuxDocTest "Name of the test set", (fix = bool),
raw\"\"\"
docstring content here
\"\"\"
This macro is used to define a doctest from within a testfile.
The first argument is the name of the test set introduced for the doctest.
`bool` may be any expression that evaluates to a boolean value (e.g. `true` or `false`),
and is used to determine whether the doctest should be fixed automatically or not.
This macro is dependent on the correct usage of commas and parentheses. Please refer to the
example above.
"""
macro _AuxDocTest(data::Expr)
@assert data.head == :tuple
@assert length(data.args) == 3
testset = data.args[1]
@assert data.args[2].head == :(=)
@assert data.args[2].args[1] == :fix
fix = data.args[2].args[2]
docstring = data.args[3]
@assert docstring.head == :macrocall
@assert docstring.args[1] == Symbol("@raw_str")
module_name = Symbol("AuxDocTest_", lstrip(string(gensym()), '#'))
logging_flag_var = Symbol("logging_flag_", gensym())
result = Expr(
:toplevel,
:(import Documenter),
:(
module $(esc(module_name))
@doc $(docstring) function dummy_placeholder end
end # module
),
# temporarily disable GC logging to avoid glitches in the doctests
if VERSION >= v"1.8.0"
(
if isdefined(GC, :logging_enabled)
esc(
quote
$(logging_flag_var) = GC.logging_enabled()
GC.enable_logging(false)
end,
)
else
esc(:(GC.enable_logging(false)))
end
)
else
nothing
end,
esc(
quote
Documenter.doctest(
nothing,
[$(module_name)];
fix=$(fix),
testset=$(testset),
doctestfilters=[
r"(?:^.*Warning: .* is deprecated, use .* instead.\n.*\n.*Core.*\n)?"m, # removes deprecation warnings
],
)
end,
),
if VERSION >= v"1.8.0"
(
if isdefined(GC, :logging_enabled)
esc(
quote
GC.enable_logging($(logging_flag_var))
end,
)
else
esc(:(GC.enable_logging(true)))
end
)
else
nothing
end,
)
Meta.replace_sourceloc!(__source__, result)
return result
end
@doc raw"""
test_module(path::AbstractString; new::Bool = true, timed::Bool=false, ignore=[])
Run the Oscar tests in `path`:
- if `path` is relative then it will be set to `<oscardir>/test/<path>`
- if `path` is a directory, run all test files in that directory and below
- if `path` or `path.jl` is a file, run this file
If a directory contains a `runtests.jl` file only this file will be executed, otherwise
all files will be included independently.
The optional parameter `new` takes the values `false` and `true` (default). If
`true`, then the tests are run in a new session, otherwise the currently active
session is used.
With the optional parameter `timed` the function will return a dict mapping file
names to a named tuple with compilation times and allocations.
This only works for `new=false`.
The parameter `ignore` can be used to pass a list of `String` or `Regex` patterns.
Test files or folders matching these will be skipped. Strings will be compared as
suffixes.
This only works for `new=false`.
For experimental modules, use [`test_experimental_module`](@ref) instead.
"""
function test_module(path::AbstractString; new::Bool=true, timed::Bool=false, tempproject::Bool=true, ignore=[])
with_unicode(false) do
julia_exe = Base.julia_cmd()
project_path = Base.active_project()
if !isabspath(path)
if !startswith(path, "test")
path = joinpath("test", path)
end
rel_test_path = normpath(path)
path = joinpath(oscardir, rel_test_path)
end
if new
@req isempty(ignore) && !timed "The `timed` and `ignore` options only work for `new=false`."
cmd = """
using Test;
using Oscar;
Hecke.assertions(true);
Oscar.test_module("$path"; new=false);
"""
@info("spawning ", `$julia_exe --project=$project_path -e \"$cmd\"`)
run(`$julia_exe --project=$project_path -e $cmd`)
else
testlist = _gather_tests(path; ignore=ignore)
@req !isempty(testlist) "no such file or directory: $path[.jl]"
use_ctime = timed && VERSION >= v"1.9.0-DEV"
if use_ctime
Base.cumulative_compile_timing(true)
end
stats = Dict{String,NamedTuple}()
if tempproject
# we preserve the old load path
oldloadpath = copy(LOAD_PATH)
# make a copy of the test environment to make sure any existing manifest doesn't interfere
tmpproj = joinpath(mktempdir(), "Project.toml")
cp(joinpath(Oscar.oscardir, "test", "Project.toml"), tmpproj)
# activate the temporary project
_setactiveproject(tmpproj)
# and make sure the current project is still available to allow e.g. `using Oscar`
pushfirst!(LOAD_PATH, dirname(project_path))
Pkg.resolve()
else
@req isdefined(Base.Main, :Test) "You need to do \"using Test\""
end
try
for entry in testlist
dir = dirname(entry)
if isfile(joinpath(dir, "setup_tests.jl"))
Base.include(identity, Main, joinpath(dir, "setup_tests.jl"))
end
if timed
push!(stats, _timed_include(entry; use_ctime=use_ctime))
else
Base.include(identity, Main, entry)
end
end
finally
# restore load path and project
if tempproject
copy!(LOAD_PATH, oldloadpath)
_setactiveproject(project_path)
end
end
if timed
use_ctime && Base.cumulative_compile_timing(false)
return stats
else
return nothing
end
end
end
end
@doc raw"""
test_experimental_module(project::AbstractString; file::AbstractString="",
new::Bool=true, timed::Bool=false, ignore=[])
Run the Oscar tests in `experimental/<project>/test/<path>`:
- if `path` is empty then all tests in that module are run, either via `runtests.jl` or directly.
- if `path` or `path.jl` is a file in that directory only this file is run.
The default is to run the entire test suite of the module `project`.
The optional parameter `new` takes the values `false` and `true` (default). If
`true`, then the tests are run in a new session, otherwise the currently active
session is used.
With the optional parameter `timed` the function will return a dict mapping file
names to a named tuple with compilation times and allocations.
This only works for `new=false`.
The parameter `ignore` can be used to pass a list of `String` or `Regex` patterns.
Test files or folders matching these will be skipped. Strings will be compared as
suffixes.
This only works for `new=false`.
"""
function test_experimental_module(
project::AbstractString; file::AbstractString="", new::Bool=true, timed::Bool=false, ignore=[]
)
test_file = "../experimental/$project/test/$file"
test_module(test_file; new, timed=timed, ignore=ignore)
end