35 changes: 31 additions & 4 deletions llvm/utils/lit/lit/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,10 @@ def main(builtinParameters = {}):
group.add_option("", "--xunit-xml-output", dest="xunit_output_file",
help=("Write XUnit-compatible XML test reports to the"
" specified file"), default=None)
group.add_option("", "--timeout", dest="maxIndividualTestTime",
help="Maximum time to spend running a single test (in seconds)."
"0 means no time limit. [Default: 0]",
type=int, default=None)
parser.add_option_group(group)

group = OptionGroup(parser, "Test Selection")
Expand Down Expand Up @@ -275,6 +279,14 @@ def main(builtinParameters = {}):
name,val = entry.split('=', 1)
userParams[name] = val

# Decide what the requested maximum indvidual test time should be
if opts.maxIndividualTestTime != None:
maxIndividualTestTime = opts.maxIndividualTestTime
else:
# Default is zero
maxIndividualTestTime = 0


# Create the global config object.
litConfig = lit.LitConfig.LitConfig(
progname = os.path.basename(sys.argv[0]),
Expand All @@ -287,12 +299,26 @@ def main(builtinParameters = {}):
debug = opts.debug,
isWindows = isWindows,
params = userParams,
config_prefix = opts.configPrefix)
config_prefix = opts.configPrefix,
maxIndividualTestTime = maxIndividualTestTime)

# Perform test discovery.
run = lit.run.Run(litConfig,
lit.discovery.find_tests_for_inputs(litConfig, inputs))

# After test discovery the configuration might have changed
# the maxIndividualTestTime. If we explicitly set this on the
# command line then override what was set in the test configuration
if opts.maxIndividualTestTime != None:
if opts.maxIndividualTestTime != litConfig.maxIndividualTestTime:
litConfig.note(('The test suite configuration requested an individual'
' test timeout of {0} seconds but a timeout of {1} seconds was'
' requested on the command line. Forcing timeout to be {1}'
' seconds')
.format(litConfig.maxIndividualTestTime,
opts.maxIndividualTestTime))
litConfig.maxIndividualTestTime = opts.maxIndividualTestTime

if opts.showSuites or opts.showTests:
# Aggregate the tests by suite.
suitesAndTests = {}
Expand Down Expand Up @@ -377,7 +403,6 @@ def main(builtinParameters = {}):
extra = ' of %d' % numTotalTests
header = '-- Testing: %d%s tests, %d threads --'%(len(run.tests), extra,
opts.numThreads)

progressBar = None
if not opts.quiet:
if opts.succinct and opts.useProgressBar:
Expand Down Expand Up @@ -422,7 +447,8 @@ def main(builtinParameters = {}):
('Failing Tests', lit.Test.FAIL),
('Unresolved Tests', lit.Test.UNRESOLVED),
('Unsupported Tests', lit.Test.UNSUPPORTED),
('Expected Failing Tests', lit.Test.XFAIL)):
('Expected Failing Tests', lit.Test.XFAIL),
('Timed Out Tests', lit.Test.TIMEOUT)):
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
(lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
continue
Expand All @@ -447,7 +473,8 @@ def main(builtinParameters = {}):
('Unsupported Tests ', lit.Test.UNSUPPORTED),
('Unresolved Tests ', lit.Test.UNRESOLVED),
('Unexpected Passes ', lit.Test.XPASS),
('Unexpected Failures', lit.Test.FAIL)):
('Unexpected Failures', lit.Test.FAIL),
('Individual Timeouts', lit.Test.TIMEOUT)):
if opts.quiet and not code.isFailure:
continue
N = len(byCode.get(code,[]))
Expand Down
92 changes: 86 additions & 6 deletions llvm/utils/lit/lit/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import signal
import subprocess
import sys
import threading

def to_bytes(str):
# Encode to UTF-8 to get binary data.
Expand Down Expand Up @@ -157,26 +158,83 @@ def printHistogram(items, title = 'Items'):
pDigits, pfDigits, i*barH, pDigits, pfDigits, (i+1)*barH,
'*'*w, ' '*(barW-w), cDigits, len(row), cDigits, len(items)))

class ExecuteCommandTimeoutException(Exception):
def __init__(self, msg, out, err, exitCode):
assert isinstance(msg, str)
assert isinstance(out, str)
assert isinstance(err, str)
assert isinstance(exitCode, int)
self.msg = msg
self.out = out
self.err = err
self.exitCode = exitCode

# Close extra file handles on UNIX (on Windows this cannot be done while
# also redirecting input).
kUseCloseFDs = not (platform.system() == 'Windows')
def executeCommand(command, cwd=None, env=None, input=None):
def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
"""
Execute command ``command`` (list of arguments or string)
with
* working directory ``cwd`` (str), use None to use the current
working directory
* environment ``env`` (dict), use None for none
* Input to the command ``input`` (str), use string to pass
no input.
* Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
Returns a tuple (out, err, exitCode) where
* ``out`` (str) is the standard output of running the command
* ``err`` (str) is the standard error of running the command
* ``exitCode`` (int) is the exitCode of running the command
If the timeout is hit an ``ExecuteCommandTimeoutException``
is raised.
"""
p = subprocess.Popen(command, cwd=cwd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env, close_fds=kUseCloseFDs)
out,err = p.communicate(input=input)
exitCode = p.wait()
timerObject = None
# FIXME: Because of the way nested function scopes work in Python 2.x we
# need to use a reference to a mutable object rather than a plain
# bool. In Python 3 we could use the "nonlocal" keyword but we need
# to support Python 2 as well.
hitTimeOut = [False]
try:
if timeout > 0:
def killProcess():
# We may be invoking a shell so we need to kill the
# process and all its children.
hitTimeOut[0] = True
killProcessAndChildren(p.pid)

# Detect Ctrl-C in subprocess.
if exitCode == -signal.SIGINT:
raise KeyboardInterrupt
timerObject = threading.Timer(timeout, killProcess)
timerObject.start()

out,err = p.communicate(input=input)
exitCode = p.wait()
finally:
if timerObject != None:
timerObject.cancel()

# Ensure the resulting output is always of string type.
out = convert_string(out)
err = convert_string(err)

if hitTimeOut[0]:
raise ExecuteCommandTimeoutException(
msg='Reached timeout of {} seconds'.format(timeout),
out=out,
err=err,
exitCode=exitCode
)

# Detect Ctrl-C in subprocess.
if exitCode == -signal.SIGINT:
raise KeyboardInterrupt

return out, err, exitCode

def usePlatformSdkOnDarwin(config, lit_config):
Expand All @@ -195,3 +253,25 @@ def usePlatformSdkOnDarwin(config, lit_config):
sdk_path = out
lit_config.note('using SDKROOT: %r' % sdk_path)
config.environment['SDKROOT'] = sdk_path

def killProcessAndChildren(pid):
"""
This function kills a process with ``pid`` and all its
running children (recursively). It is currently implemented
using the psutil module which provides a simple platform
neutral implementation.
TODO: Reimplement this without using psutil so we can
remove our dependency on it.
"""
import psutil
try:
psutilProc = psutil.Process(pid)
for child in psutilProc.children(recursive=True):
try:
child.kill()
except psutil.NoSuchProcess:
pass
psutilProc.kill()
except psutil.NoSuchProcess:
pass
35 changes: 35 additions & 0 deletions llvm/utils/lit/tests/Inputs/googletest-timeout/DummySubDir/OneTest
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/usr/bin/env python

import sys
import time

if len(sys.argv) != 2:
raise ValueError("unexpected number of args")

if sys.argv[1] == "--gtest_list_tests":
print("""\
FirstTest.
subTestA
subTestB
subTestC
""")
sys.exit(0)
elif not sys.argv[1].startswith("--gtest_filter="):
raise ValueError("unexpected argument: %r" % (sys.argv[1]))

test_name = sys.argv[1].split('=',1)[1]
if test_name == 'FirstTest.subTestA':
print('I am subTest A, I PASS')
print('[ PASSED ] 1 test.')
sys.exit(0)
elif test_name == 'FirstTest.subTestB':
print('I am subTest B, I am slow')
time.sleep(6)
print('[ PASSED ] 1 test.')
sys.exit(0)
elif test_name == 'FirstTest.subTestC':
print('I am subTest C, I will hang')
while True:
pass
else:
raise SystemExit("error: invalid test name: %r" % (test_name,))
9 changes: 9 additions & 0 deletions llvm/utils/lit/tests/Inputs/googletest-timeout/lit.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import lit.formats
config.name = 'googletest-timeout'
config.test_format = lit.formats.GoogleTest('DummySubDir', 'Test')

configSetTimeout = lit_config.params.get('set_timeout', '0')

if configSetTimeout == '1':
# Try setting the max individual test time in the configuration
lit_config.maxIndividualTestTime = 1
10 changes: 10 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-timeout/infinite_loop.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# RUN: %{python} %s
from __future__ import print_function

import time
import sys

print("Running infinite loop")
sys.stdout.flush() # Make sure the print gets flushed so it appears in lit output.
while True:
pass
32 changes: 32 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-timeout/lit.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# -*- Python -*-
import os
import sys

import lit.formats

config.name = 'per_test_timeout'

shellType = lit_config.params.get('external', '1')

if shellType == '0':
lit_config.note('Using internal shell')
externalShell = False
else:
lit_config.note('Using external shell')
externalShell = True

configSetTimeout = lit_config.params.get('set_timeout', '0')

if configSetTimeout == '1':
# Try setting the max individual test time in the configuration
lit_config.maxIndividualTestTime = 1

config.test_format = lit.formats.ShTest(execute_external=externalShell)
config.suffixes = ['.py']

config.test_source_root = os.path.dirname(__file__)
config.test_exec_root = config.test_source_root
config.target_triple = '(unused)'
src_root = os.path.join(config.test_source_root, '..')
config.environment['PYTHONPATH'] = src_root
config.substitutions.append(('%{python}', sys.executable))
24 changes: 24 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-timeout/quick_then_slow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# RUN: %{python} %s quick
# RUN: %{python} %s slow
from __future__ import print_function

import time
import sys

if len(sys.argv) != 2:
print("Wrong number of args")
sys.exit(1)

mode = sys.argv[1]

if mode == 'slow':
print("Running in slow mode")
sys.stdout.flush() # Make sure the print gets flushed so it appears in lit output.
time.sleep(6)
sys.exit(0)
elif mode == 'quick':
print("Running in quick mode")
sys.exit(0)
else:
print("Unrecognised mode {}".format(mode))
sys.exit(1)
6 changes: 6 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-timeout/short.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# RUN: %{python} %s
from __future__ import print_function

import sys

print("short program")
9 changes: 9 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-timeout/slow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# RUN: %{python} %s
from __future__ import print_function

import time
import sys

print("Running slow program")
sys.stdout.flush() # Make sure the print gets flushed so it appears in lit output.
time.sleep(6)
29 changes: 29 additions & 0 deletions llvm/utils/lit/tests/googletest-timeout.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# REQUIRES: python-psutil

# Check that the per test timeout is enforced when running GTest tests.
#
# RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout --timeout=1 > %t.cmd.out
# RUN: FileCheck < %t.cmd.out %s

# Check that the per test timeout is enforced when running GTest tests via
# the configuration file
#
# RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
# RUN: --param set_timeout=1 > %t.cfgset.out 2> %t.cfgset.err
# RUN: FileCheck < %t.cfgset.out %s

# CHECK: -- Testing:
# CHECK: PASS: googletest-timeout :: DummySubDir/OneTest/FirstTest.subTestA
# CHECK: TIMEOUT: googletest-timeout :: DummySubDir/OneTest/FirstTest.subTestB
# CHECK: TIMEOUT: googletest-timeout :: DummySubDir/OneTest/FirstTest.subTestC
# CHECK: Expected Passes : 1
# CHECK: Individual Timeouts: 2

# Test per test timeout via a config file and on the command line.
# The value set on the command line should override the config file.
# RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
# RUN: --param set_timeout=1 --timeout=2 > %t.cmdover.out 2> %t.cmdover.err
# RUN: FileCheck < %t.cmdover.out %s
# RUN: FileCheck --check-prefix=CHECK-CMDLINE-OVERRIDE-ERR < %t.cmdover.err %s

# CHECK-CMDLINE-OVERRIDE-ERR: Forcing timeout to be 2 seconds
9 changes: 9 additions & 0 deletions llvm/utils/lit/tests/lit.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,12 @@ if lit_config.params.get('check-coverage', None):
# Add a feature to detect the Python version.
config.available_features.add("python%d.%d" % (sys.version_info[0],
sys.version_info[1]))

# Add a feature to detect if psutil is available
try:
import psutil
lit_config.note('Found python psutil module')
config.available_features.add("python-psutil")
except ImportError:
lit_config.warning('Could not import psutil. Some tests will be skipped and'
' the --timeout command line argument will not work.')
116 changes: 116 additions & 0 deletions llvm/utils/lit/tests/shtest-timeout.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# REQUIRES: python-psutil

# Test per test timeout using external shell
# RUN: not %{lit} \
# RUN: %{inputs}/shtest-timeout/infinite_loop.py \
# RUN: %{inputs}/shtest-timeout/quick_then_slow.py \
# RUN: %{inputs}/shtest-timeout/short.py \
# RUN: %{inputs}/shtest-timeout/slow.py \
# RUN: -j 1 -v --debug --timeout 1 --param external=1 > %t.extsh.out 2> %t.extsh.err
# RUN: FileCheck --check-prefix=CHECK-OUT-COMMON < %t.extsh.out %s
# RUN: FileCheck --check-prefix=CHECK-EXTSH-ERR < %t.extsh.err %s
#
# CHECK-EXTSH-ERR: Using external shell

# Test per test timeout using internal shell
# RUN: not %{lit} \
# RUN: %{inputs}/shtest-timeout/infinite_loop.py \
# RUN: %{inputs}/shtest-timeout/quick_then_slow.py \
# RUN: %{inputs}/shtest-timeout/short.py \
# RUN: %{inputs}/shtest-timeout/slow.py \
# RUN: -j 1 -v --debug --timeout 1 --param external=0 > %t.intsh.out 2> %t.intsh.err
# RUN: FileCheck --check-prefix=CHECK-OUT-COMMON < %t.intsh.out %s
# RUN: FileCheck --check-prefix=CHECK-INTSH-OUT < %t.intsh.out %s
# RUN: FileCheck --check-prefix=CHECK-INTSH-ERR < %t.intsh.err %s
#
# CHECK-INTSH-OUT: TIMEOUT: per_test_timeout :: infinite_loop.py
# CHECK-INTSH-OUT: Command 0 Reached Timeout: True
# CHECK-INTSH-OUT: Command 0 Output:
# CHECK-INTSH-OUT-NEXT: Running infinite loop


# CHECK-INTSH-OUT: TIMEOUT: per_test_timeout :: quick_then_slow.py
# CHECK-INTSH-OUT: Timeout: Reached timeout of 1 seconds
# CHECK-INTSH-OUT: Command Output
# CHECK-INTSH-OUT: Command 0 Reached Timeout: False
# CHECK-INTSH-OUT: Command 0 Output:
# CHECK-INTSH-OUT-NEXT: Running in quick mode
# CHECK-INTSH-OUT: Command 1 Reached Timeout: True
# CHECK-INTSH-OUT: Command 1 Output:
# CHECK-INTSH-OUT-NEXT: Running in slow mode

# CHECK-INTSH-OUT: TIMEOUT: per_test_timeout :: slow.py
# CHECK-INTSH-OUT: Command 0 Reached Timeout: True
# CHECK-INTSH-OUT: Command 0 Output:
# CHECK-INTSH-OUT-NEXT: Running slow program

# CHECK-INTSH-ERR: Using internal shell

# Test per test timeout set via a config file rather than on the command line
# RUN: not %{lit} \
# RUN: %{inputs}/shtest-timeout/infinite_loop.py \
# RUN: %{inputs}/shtest-timeout/quick_then_slow.py \
# RUN: %{inputs}/shtest-timeout/short.py \
# RUN: %{inputs}/shtest-timeout/slow.py \
# RUN: -j 1 -v --debug --param external=0 \
# RUN: --param set_timeout=1 > %t.cfgset.out 2> %t.cfgset.err
# RUN: FileCheck --check-prefix=CHECK-OUT-COMMON < %t.cfgset.out %s
# RUN: FileCheck --check-prefix=CHECK-CFGSET-ERR < %t.cfgset.err %s
#
# CHECK-CFGSET-ERR: Using internal shell

# CHECK-OUT-COMMON: TIMEOUT: per_test_timeout :: infinite_loop.py
# CHECK-OUT-COMMON: Timeout: Reached timeout of 1 seconds
# CHECK-OUT-COMMON: Command {{([0-9]+ )?}}Output
# CHECK-OUT-COMMON: Running infinite loop

# CHECK-OUT-COMMON: TIMEOUT: per_test_timeout :: quick_then_slow.py
# CHECK-OUT-COMMON: Timeout: Reached timeout of 1 seconds
# CHECK-OUT-COMMON: Command {{([0-9]+ )?}}Output
# CHECK-OUT-COMMON: Running in quick mode
# CHECK-OUT-COMMON: Running in slow mode

# CHECK-OUT-COMMON: PASS: per_test_timeout :: short.py

# CHECK-OUT-COMMON: TIMEOUT: per_test_timeout :: slow.py
# CHECK-OUT-COMMON: Timeout: Reached timeout of 1 seconds
# CHECK-OUT-COMMON: Command {{([0-9]+ )?}}Output
# CHECK-OUT-COMMON: Running slow program

# CHECK-OUT-COMMON: Expected Passes{{ *}}: 1
# CHECK-OUT-COMMON: Individual Timeouts{{ *}}: 3

# Test per test timeout via a config file and on the command line.
# The value set on the command line should override the config file.
# RUN: not %{lit} \
# RUN: %{inputs}/shtest-timeout/infinite_loop.py \
# RUN: %{inputs}/shtest-timeout/quick_then_slow.py \
# RUN: %{inputs}/shtest-timeout/short.py \
# RUN: %{inputs}/shtest-timeout/slow.py \
# RUN: -j 1 -v --debug --param external=0 \
# RUN: --param set_timeout=1 --timeout=2 > %t.cmdover.out 2> %t.cmdover.err
# RUN: FileCheck --check-prefix=CHECK-CMDLINE-OVERRIDE-OUT < %t.cmdover.out %s
# RUN: FileCheck --check-prefix=CHECK-CMDLINE-OVERRIDE-ERR < %t.cmdover.err %s

# CHECK-CMDLINE-OVERRIDE-ERR: Forcing timeout to be 2 seconds

# CHECK-CMDLINE-OVERRIDE-OUT: TIMEOUT: per_test_timeout :: infinite_loop.py
# CHECK-CMDLINE-OVERRIDE-OUT: Timeout: Reached timeout of 2 seconds
# CHECK-CMDLINE-OVERRIDE-OUT: Command {{([0-9]+ )?}}Output
# CHECK-CMDLINE-OVERRIDE-OUT: Running infinite loop

# CHECK-CMDLINE-OVERRIDE-OUT: TIMEOUT: per_test_timeout :: quick_then_slow.py
# CHECK-CMDLINE-OVERRIDE-OUT: Timeout: Reached timeout of 2 seconds
# CHECK-CMDLINE-OVERRIDE-OUT: Command {{([0-9]+ )?}}Output
# CHECK-CMDLINE-OVERRIDE-OUT: Running in quick mode
# CHECK-CMDLINE-OVERRIDE-OUT: Running in slow mode

# CHECK-CMDLINE-OVERRIDE-OUT: PASS: per_test_timeout :: short.py

# CHECK-CMDLINE-OVERRIDE-OUT: TIMEOUT: per_test_timeout :: slow.py
# CHECK-CMDLINE-OVERRIDE-OUT: Timeout: Reached timeout of 2 seconds
# CHECK-CMDLINE-OVERRIDE-OUT: Command {{([0-9]+ )?}}Output
# CHECK-CMDLINE-OVERRIDE-OUT: Running slow program

# CHECK-CMDLINE-OVERRIDE-OUT: Expected Passes{{ *}}: 1
# CHECK-CMDLINE-OVERRIDE-OUT: Individual Timeouts{{ *}}: 3