-
Notifications
You must be signed in to change notification settings - Fork 10.7k
/
format.py
426 lines (358 loc) · 17.5 KB
/
format.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
# ===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ===----------------------------------------------------------------------===##
import contextlib
import io
import lit
import lit.formats
import os
import pipes
import re
import shutil
def _getTempPaths(test):
"""
Return the values to use for the %T and %t substitutions, respectively.
The difference between this and Lit's default behavior is that we guarantee
that %T is a path unique to the test being run.
"""
tmpDir, _ = lit.TestRunner.getTempPaths(test)
_, testName = os.path.split(test.getExecPath())
tmpDir = os.path.join(tmpDir, testName + ".dir")
tmpBase = os.path.join(tmpDir, "t")
return tmpDir, tmpBase
def _checkBaseSubstitutions(substitutions):
substitutions = [s for (s, _) in substitutions]
for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
assert s in substitutions, "Required substitution {} was not provided".format(s)
def _parseLitOutput(fullOutput):
"""
Parse output of a Lit ShTest to extract the actual output of the contained commands.
This takes output of the form
$ ":" "RUN: at line 11"
$ "echo" "OUTPUT1"
# command output:
OUTPUT1
$ ":" "RUN: at line 12"
$ "echo" "OUTPUT2"
# command output:
OUTPUT2
and returns a string containing
OUTPUT1
OUTPUT2
as-if the commands had been run directly. This is a workaround for the fact
that Lit doesn't let us execute ShTest and retrieve the raw output without
injecting additional Lit output around it.
"""
parsed = ''
for output in re.split('[$]\s*":"\s*"RUN: at line \d+"', fullOutput):
if output: # skip blank lines
commandOutput = re.search("# command output:\n(.+)\n$", output, flags=re.DOTALL)
if commandOutput:
parsed += commandOutput.group(1)
return parsed
def _executeScriptInternal(test, litConfig, commands):
"""
Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands)
TODO: This really should be easier to access from Lit itself
"""
parsedCommands = parseScript(test, preamble=commands)
_, tmpBase = _getTempPaths(test)
execDir = os.path.dirname(test.getExecPath())
res = lit.TestRunner.executeScriptInternal(
test, litConfig, tmpBase, parsedCommands, execDir
)
if isinstance(res, lit.Test.Result): # Handle failure to parse the Lit test
res = ("", res.output, 127, None)
(out, err, exitCode, timeoutInfo) = res
# TODO: As a temporary workaround until https://reviews.llvm.org/D81892 lands, manually
# split any stderr output that is included in stdout. It shouldn't be there, but
# the Lit internal shell conflates stderr and stdout.
conflatedErrorOutput = re.search("(# command stderr:.+$)", out, flags=re.DOTALL)
if conflatedErrorOutput:
conflatedErrorOutput = conflatedErrorOutput.group(0)
out = out[: -len(conflatedErrorOutput)]
err += conflatedErrorOutput
return (out, err, exitCode, timeoutInfo, parsedCommands)
def parseScript(test, preamble):
"""
Extract the script from a test, with substitutions applied.
Returns a list of commands ready to be executed.
- test
The lit.Test to parse.
- preamble
A list of commands to perform before any command in the test.
These commands can contain unexpanded substitutions, but they
must not be of the form 'RUN:' -- they must be proper commands
once substituted.
"""
# Get the default substitutions
tmpDir, tmpBase = _getTempPaths(test)
substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase)
# Check base substitutions and add the %{build} and %{run} convenience substitutions
_checkBaseSubstitutions(substitutions)
substitutions.append(
("%{build}", "%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe")
)
substitutions.append(("%{run}", "%{exec} %t.exe"))
# Parse the test file, including custom directives
additionalCompileFlags = []
fileDependencies = []
parsers = [
lit.TestRunner.IntegratedTestKeywordParser(
"FILE_DEPENDENCIES:",
lit.TestRunner.ParserKind.LIST,
initial_value=fileDependencies,
),
lit.TestRunner.IntegratedTestKeywordParser(
"ADDITIONAL_COMPILE_FLAGS:",
lit.TestRunner.ParserKind.LIST,
initial_value=additionalCompileFlags,
),
]
# Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first
# class support for conditional keywords in Lit, which would allow evaluating arbitrary
# Lit boolean expressions instead.
for feature in test.config.available_features:
parser = lit.TestRunner.IntegratedTestKeywordParser(
"ADDITIONAL_COMPILE_FLAGS({}):".format(feature),
lit.TestRunner.ParserKind.LIST,
initial_value=additionalCompileFlags,
)
parsers.append(parser)
scriptInTest = lit.TestRunner.parseIntegratedTestScript(
test, additional_parsers=parsers, require_script=not preamble
)
if isinstance(scriptInTest, lit.Test.Result):
return scriptInTest
script = []
# For each file dependency in FILE_DEPENDENCIES, inject a command to copy
# that file to the execution directory. Execute the copy from %S to allow
# relative paths from the test directory.
for dep in fileDependencies:
script += ["%dbg(SETUP) cd %S && cp {} %T".format(dep)]
script += preamble
script += scriptInTest
# Add compile flags specified with ADDITIONAL_COMPILE_FLAGS.
substitutions = [
(s, x + " " + " ".join(additionalCompileFlags))
if s == "%{compile_flags}"
else (s, x)
for (s, x) in substitutions
]
# Perform substitutions in the script itself.
script = lit.TestRunner.applySubstitutions(
script, substitutions, recursion_limit=test.config.recursiveExpansionLimit
)
return script
class CxxStandardLibraryTest(lit.formats.FileBasedTest):
"""
Lit test format for the C++ Standard Library conformance test suite.
This test format is based on top of the ShTest format -- it basically
creates a shell script performing the right operations (compile/link/run)
based on the extension of the test file it encounters. It supports files
with the following extensions:
FOO.pass.cpp - Compiles, links and runs successfully
FOO.pass.mm - Same as .pass.cpp, but for Objective-C++
FOO.compile.pass.cpp - Compiles successfully, link and run not attempted
FOO.compile.pass.mm - Same as .compile.pass.cpp, but for Objective-C++
FOO.compile.fail.cpp - Does not compile successfully
FOO.link.pass.cpp - Compiles and links successfully, run not attempted
FOO.link.pass.mm - Same as .link.pass.cpp, but for Objective-C++
FOO.link.fail.cpp - Compiles successfully, but fails to link
FOO.sh.<anything> - A builtin Lit Shell test
FOO.gen.<anything> - A .sh test that generates one or more Lit tests on the
fly. Executing this test must generate one or more files
as expected by LLVM split-file, and each generated file
leads to a separate Lit test that runs that file as
defined by the test format. This can be used to generate
multiple Lit tests from a single source file, which is
useful for testing repetitive properties in the library.
Be careful not to abuse this since this is not a replacement
for usual code reuse techniques.
FOO.verify.cpp - Compiles with clang-verify. This type of test is
automatically marked as UNSUPPORTED if the compiler
does not support Clang-verify.
Substitution requirements
===============================
The test format operates by assuming that each test's configuration provides
the following substitutions, which it will reuse in the shell scripts it
constructs:
%{cxx} - A command that can be used to invoke the compiler
%{compile_flags} - Flags to use when compiling a test case
%{link_flags} - Flags to use when linking a test case
%{flags} - Flags to use either when compiling or linking a test case
%{exec} - A command to prefix the execution of executables
Note that when building an executable (as opposed to only compiling a source
file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used
in the same command line. In other words, the test format doesn't perform
separate compilation and linking steps in this case.
Additional supported directives
===============================
In addition to everything that's supported in Lit ShTests, this test format
also understands the following directives inside test files:
// FILE_DEPENDENCIES: file, directory, /path/to/file
This directive expresses that the test requires the provided files
or directories in order to run. An example is a test that requires
some test input stored in a data file. When a test file contains
such a directive, this test format will collect them and copy them
to the directory represented by %T. The intent is that %T contains
all the inputs necessary to run the test, such that e.g. execution
on a remote host can be done by simply copying %T to the host.
// ADDITIONAL_COMPILE_FLAGS: flag1, flag2, flag3
This directive will cause the provided flags to be added to the
%{compile_flags} substitution for the test that contains it. This
allows adding special compilation flags without having to use a
.sh.cpp test, which would be more powerful but perhaps overkill.
Additional provided substitutions and features
==============================================
The test format will define the following substitutions for use inside tests:
%{build}
Expands to a command-line that builds the current source
file with the %{flags}, %{compile_flags} and %{link_flags}
substitutions, and that produces an executable named %t.exe.
%{run}
Equivalent to `%{exec} %t.exe`. This is intended to be used
in conjunction with the %{build} substitution.
"""
def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig):
SUPPORTED_SUFFIXES = [
"[.]pass[.]cpp$",
"[.]pass[.]mm$",
"[.]compile[.]pass[.]cpp$",
"[.]compile[.]pass[.]mm$",
"[.]compile[.]fail[.]cpp$",
"[.]link[.]pass[.]cpp$",
"[.]link[.]pass[.]mm$",
"[.]link[.]fail[.]cpp$",
"[.]sh[.][^.]+$",
"[.]gen[.][^.]+$",
"[.]verify[.]cpp$",
"[.]fail[.]cpp$",
]
sourcePath = testSuite.getSourcePath(pathInSuite)
filename = os.path.basename(sourcePath)
# Ignore dot files, excluded tests and tests with an unsupported suffix
hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES])
if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename):
return
# If this is a generated test, run the generation step and add
# as many Lit tests as necessary.
if re.search('[.]gen[.][^.]+$', filename):
for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig):
yield test
else:
yield lit.Test.Test(testSuite, pathInSuite, localConfig)
def execute(self, test, litConfig):
VERIFY_FLAGS = (
"-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0"
)
supportsVerify = "verify-support" in test.config.available_features
filename = test.path_in_suite[-1]
if re.search("[.]sh[.][^.]+$", filename):
steps = [] # The steps are already in the script
return self._executeShTest(test, litConfig, steps)
elif filename.endswith(".compile.pass.cpp") or filename.endswith(
".compile.pass.mm"
):
steps = [
"%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
]
return self._executeShTest(test, litConfig, steps)
elif filename.endswith(".compile.fail.cpp"):
steps = [
"%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only"
]
return self._executeShTest(test, litConfig, steps)
elif filename.endswith(".link.pass.cpp") or filename.endswith(".link.pass.mm"):
steps = [
"%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe"
]
return self._executeShTest(test, litConfig, steps)
elif filename.endswith(".link.fail.cpp"):
steps = [
"%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o",
"%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe",
]
return self._executeShTest(test, litConfig, steps)
elif filename.endswith(".verify.cpp"):
if not supportsVerify:
return lit.Test.Result(
lit.Test.UNSUPPORTED,
"Test {} requires support for Clang-verify, which isn't supported by the compiler".format(
test.getFullName()
),
)
steps = [
# Note: Use -Wno-error to make sure all diagnostics are not treated as errors,
# which doesn't make sense for clang-verify tests.
"%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(
VERIFY_FLAGS
)
]
return self._executeShTest(test, litConfig, steps)
# Make sure to check these ones last, since they will match other
# suffixes above too.
elif filename.endswith(".pass.cpp") or filename.endswith(".pass.mm"):
steps = [
"%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe",
"%dbg(EXECUTED AS) %{exec} %t.exe",
]
return self._executeShTest(test, litConfig, steps)
else:
return lit.Test.Result(
lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)
)
def _executeShTest(self, test, litConfig, steps):
if test.config.unsupported:
return lit.Test.Result(lit.Test.UNSUPPORTED, "Test is unsupported")
script = parseScript(test, steps)
if isinstance(script, lit.Test.Result):
return script
if litConfig.noExecute:
return lit.Test.Result(
lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS
)
else:
_, tmpBase = _getTempPaths(test)
useExternalSh = False
return lit.TestRunner._runShTest(
test, litConfig, useExternalSh, script, tmpBase
)
def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
generator = lit.Test.Test(testSuite, pathInSuite, localConfig)
# Make sure we have a directory to execute the generator test in
generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
os.makedirs(generatorExecDir, exist_ok=True)
# Run the generator test
steps = [] # Steps must already be in the script
(out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
if exitCode != 0:
raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}")
# Split the generated output into multiple files and generate one test for each file
parsed = _parseLitOutput(out)
for (subfile, content) in self._splitFile(parsed):
generatedFile = testSuite.getExecPath(pathInSuite + (subfile, ))
os.makedirs(os.path.dirname(generatedFile), exist_ok=True)
with open(generatedFile, 'w') as f:
f.write(content)
yield lit.Test.Test(testSuite, (generatedFile,), localConfig)
def _splitFile(self, input):
DELIM = r'^(//|#)---(.+)'
lines = input.splitlines()
currentFile = None
thisFileContent = []
for line in lines:
match = re.match(DELIM, line)
if match:
if currentFile is not None:
yield (currentFile, '\n'.join(thisFileContent))
currentFile = match.group(2).strip()
thisFileContent = []
assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}"
thisFileContent.append(line)
if currentFile is not None:
yield (currentFile, '\n'.join(thisFileContent))