118 changes: 103 additions & 15 deletions llvm/utils/lit/lit/Test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from xml.sax.saxutils import escape
from json import JSONEncoder

from lit.BooleanExpression import BooleanExpression

# Test result codes.

class ResultCode(object):
Expand Down Expand Up @@ -180,10 +182,24 @@ def __init__(self, suite, path_in_suite, config, file_path = None):
self.path_in_suite = path_in_suite
self.config = config
self.file_path = file_path
# A list of conditions under which this test is expected to fail. These
# can optionally be provided by test format handlers, and will be
# honored when the test result is supplied.

# A list of conditions under which this test is expected to fail.
# Each condition is a boolean expression of features and target
# triple parts. These can optionally be provided by test format
# handlers, and will be honored when the test result is supplied.
self.xfails = []

# A list of conditions that must be satisfied before running the test.
# Each condition is a boolean expression of features. All of them
# must be True for the test to run.
# FIXME should target triple parts count here too?
self.requires = []

# A list of conditions that prevent execution of the test.
# Each condition is a boolean expression of features and target
# triple parts. All of them must be False for the test to run.
self.unsupported = []

# The test result, once complete.
self.result = None

Expand All @@ -196,11 +212,16 @@ def setResult(self, result):
self.result = result

# Apply the XFAIL handling to resolve the result exit code.
if self.isExpectedToFail():
if self.result.code == PASS:
self.result.code = XPASS
elif self.result.code == FAIL:
self.result.code = XFAIL
try:
if self.isExpectedToFail():
if self.result.code == PASS:
self.result.code = XPASS
elif self.result.code == FAIL:
self.result.code = XFAIL
except ValueError as e:
# Syntax error in an XFAIL line.
self.result.code = UNRESOLVED
self.result.output = str(e)

def getFullName(self):
return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
Expand All @@ -224,24 +245,91 @@ def isExpectedToFail(self):
configuration. This check relies on the test xfails property which by
some test formats may not be computed until the test has first been
executed.
Throws ValueError if an XFAIL line has a syntax error.
"""

features = self.config.available_features
triple = getattr(self.suite.config, 'target_triple', "")

# Check if any of the xfails match an available feature or the target.
for item in self.xfails:
# If this is the wildcard, it always fails.
if item == '*':
return True

# If this is an exact match for one of the features, it fails.
if item in self.config.available_features:
return True

# If this is a part of the target triple, it fails.
if item and item in self.suite.config.target_triple:
return True
# If this is a True expression of features and target triple parts,
# it fails.
try:
if BooleanExpression.evaluate(item, features, triple):
return True
except ValueError as e:
raise ValueError('Error in XFAIL list:\n%s' % str(e))

return False

def isWithinFeatureLimits(self):
"""
isWithinFeatureLimits() -> bool
A test is within the feature limits set by run_only_tests if
1. the test's requirements ARE satisfied by the available features
2. the test's requirements ARE NOT satisfied after the limiting
features are removed from the available features
Throws ValueError if a REQUIRES line has a syntax error.
"""

if not self.config.limit_to_features:
return True # No limits. Run it.

# Check the requirements as-is (#1)
if self.getMissingRequiredFeatures():
return False

# Check the requirements after removing the limiting features (#2)
featuresMinusLimits = [f for f in self.config.available_features
if not f in self.config.limit_to_features]
if not self.getMissingRequiredFeaturesFromList(featuresMinusLimits):
return False

return True

def getMissingRequiredFeaturesFromList(self, features):
try:
return [item for item in self.requires
if not BooleanExpression.evaluate(item, features)]
except ValueError as e:
raise ValueError('Error in REQUIRES list:\n%s' % str(e))

def getMissingRequiredFeatures(self):
"""
getMissingRequiredFeatures() -> list of strings
Returns a list of features from REQUIRES that are not satisfied."
Throws ValueError if a REQUIRES line has a syntax error.
"""

features = self.config.available_features
return self.getMissingRequiredFeaturesFromList(features)

def getUnsupportedFeatures(self):
"""
getUnsupportedFeatures() -> list of strings
Returns a list of features from UNSUPPORTED that are present
in the test configuration's features or target triple.
Throws ValueError if an UNSUPPORTED line has a syntax error.
"""

features = self.config.available_features
triple = getattr(self.suite.config, 'target_triple', "")

try:
return [item for item in self.unsupported
if BooleanExpression.evaluate(item, features, triple)]
except ValueError as e:
raise ValueError('Error in UNSUPPORTED list:\n%s' % str(e))

def isEarlyTest(self):
"""
isEarlyTest() -> bool
Expand Down
156 changes: 95 additions & 61 deletions llvm/utils/lit/lit/TestRunner.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import lit.Test as Test
import lit.util
from lit.util import to_bytes, to_string
from lit.BooleanExpression import BooleanExpression

class InternalShellError(Exception):
def __init__(self, command, message):
Expand Down Expand Up @@ -746,14 +747,35 @@ class ParserKind(object):
command.
TAG: A keyword taking no value. Ex 'END.'
COMMAND: A Keyword taking a list of shell commands. Ex 'RUN:'
LIST: A keyword taking a comma separated list of value. Ex 'XFAIL:'
COMMAND: A keyword taking a list of shell commands. Ex 'RUN:'
LIST: A keyword taking a comma-separated list of values.
BOOLEAN_EXPR: A keyword taking a comma-separated list of
boolean expressions. Ex 'XFAIL:'
CUSTOM: A keyword with custom parsing semantics.
"""
TAG = 0
COMMAND = 1
LIST = 2
CUSTOM = 3
BOOLEAN_EXPR = 3
CUSTOM = 4

@staticmethod
def allowedKeywordSuffixes(value):
return { ParserKind.TAG: ['.'],
ParserKind.COMMAND: [':'],
ParserKind.LIST: [':'],
ParserKind.BOOLEAN_EXPR: [':'],
ParserKind.CUSTOM: [':', '.']
} [value]

@staticmethod
def str(value):
return { ParserKind.TAG: 'TAG',
ParserKind.COMMAND: 'COMMAND',
ParserKind.LIST: 'LIST',
ParserKind.BOOLEAN_EXPR: 'BOOLEAN_EXPR',
ParserKind.CUSTOM: 'CUSTOM'
} [value]


class IntegratedTestKeywordParser(object):
Expand All @@ -765,15 +787,18 @@ class IntegratedTestKeywordParser(object):
ParserKind.CUSTOM.
"""
def __init__(self, keyword, kind, parser=None, initial_value=None):
if not keyword.endswith('.') and not keyword.endswith(':'):
raise ValueError("keyword '%s' must end with either '.' or ':' "
% keyword)
if keyword.endswith('.') and kind in \
[ParserKind.LIST, ParserKind.COMMAND]:
raise ValueError("Keyword '%s' should end in ':'" % keyword)

elif keyword.endswith(':') and kind in [ParserKind.TAG]:
raise ValueError("Keyword '%s' should end in '.'" % keyword)
allowedSuffixes = ParserKind.allowedKeywordSuffixes(kind)
if len(keyword) == 0 or keyword[-1] not in allowedSuffixes:
if len(allowedSuffixes) == 1:
raise ValueError("Keyword '%s' of kind '%s' must end in '%s'"
% (keyword, ParserKind.str(kind),
allowedSuffixes[0]))
else:
raise ValueError("Keyword '%s' of kind '%s' must end in "
" one of '%s'"
% (keyword, ParserKind.str(kind),
' '.join(allowedSuffixes)))

if parser is not None and kind != ParserKind.CUSTOM:
raise ValueError("custom parsers can only be specified with "
"ParserKind.CUSTOM")
Expand All @@ -787,9 +812,9 @@ def __init__(self, keyword, kind, parser=None, initial_value=None):
self.parser = self._handleCommand
elif kind == ParserKind.LIST:
self.parser = self._handleList
elif kind == ParserKind.BOOLEAN_EXPR:
self.parser = self._handleBooleanExpr
elif kind == ParserKind.TAG:
if not keyword.endswith('.'):
raise ValueError("keyword '%s' should end with '.'" % keyword)
self.parser = self._handleTag
elif kind == ParserKind.CUSTOM:
if parser is None:
Expand All @@ -799,8 +824,12 @@ def __init__(self, keyword, kind, parser=None, initial_value=None):
raise ValueError("Unknown kind '%s'" % kind)

def parseLine(self, line_number, line):
self.parsed_lines += [(line_number, line)]
self.value = self.parser(line_number, line, self.value)
try:
self.parsed_lines += [(line_number, line)]
self.value = self.parser(line_number, line, self.value)
except ValueError as e:
raise ValueError(str(e) + ("\nin %s directive on test line %d" %
(self.keyword, line_number)))

def getValue(self):
return self.value
Expand Down Expand Up @@ -841,12 +870,24 @@ def _handleList(line_number, line, output):
output.extend([s.strip() for s in line.split(',')])
return output

@staticmethod
def _handleBooleanExpr(line_number, line, output):
"""A parser for BOOLEAN_EXPR type keywords"""
if output is None:
output = []
output.extend([s.strip() for s in line.split(',')])
# Evaluate each expression to verify syntax.
# We don't want any results, just the raised ValueError.
for s in output:
if s != '*':
BooleanExpression.evaluate(s, [])
return output

def parseIntegratedTestScript(test, additional_parsers=[],
require_script=True):
"""parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
'REQUIRES-ANY' and 'UNSUPPORTED' information.
and 'UNSUPPORTED' information.
If additional parsers are specified then the test is also scanned for the
keywords they specify and all matches are passed to the custom parser.
Expand All @@ -855,26 +896,23 @@ def parseIntegratedTestScript(test, additional_parsers=[],
may be returned. This can be used for test formats where the actual script
is optional or ignored.
"""
# Collect the test lines from the script.
sourcepath = test.getSourcePath()

# Install the built-in keyword parsers.
script = []
requires = []
requires_any = []
unsupported = []
builtin_parsers = [
IntegratedTestKeywordParser('RUN:', ParserKind.COMMAND,
initial_value=script),
IntegratedTestKeywordParser('XFAIL:', ParserKind.LIST,
IntegratedTestKeywordParser('XFAIL:', ParserKind.BOOLEAN_EXPR,
initial_value=test.xfails),
IntegratedTestKeywordParser('REQUIRES:', ParserKind.LIST,
initial_value=requires),
IntegratedTestKeywordParser('REQUIRES-ANY:', ParserKind.LIST,
initial_value=requires_any),
IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.LIST,
initial_value=unsupported),
IntegratedTestKeywordParser('REQUIRES:', ParserKind.BOOLEAN_EXPR,
initial_value=test.requires),
IntegratedTestKeywordParser('UNSUPPORTED:', ParserKind.BOOLEAN_EXPR,
initial_value=test.unsupported),
IntegratedTestKeywordParser('END.', ParserKind.TAG)
]
keyword_parsers = {p.keyword: p for p in builtin_parsers}

# Install user-defined additional parsers.
for parser in additional_parsers:
if not isinstance(parser, IntegratedTestKeywordParser):
raise ValueError('additional parser must be an instance of '
Expand All @@ -884,6 +922,18 @@ def parseIntegratedTestScript(test, additional_parsers=[],
% parser.keyword)
keyword_parsers[parser.keyword] = parser

# Install a helpful error-generating parser for the no-longer-supported
# REQUIRES-ANY: keyword, if no other parser for it exists.
if 'REQUIRES-ANY:' not in keyword_parsers:
def requires_any_error_parser(line_number, line, output):
raise ValueError('`REQUIRES-ANY: a, b, c` not supported. Use '
'`REQUIRES: a || b || c` instead.')
parser = IntegratedTestKeywordParser('REQUIRES-ANY:', ParserKind.CUSTOM,
requires_any_error_parser)
keyword_parsers[parser.keyword] = parser

# Collect the test lines from the script.
sourcepath = test.getSourcePath()
for line_number, command_type, ln in \
parseIntegratedTestScriptCommands(sourcepath,
keyword_parsers.keys()):
Expand All @@ -901,46 +951,30 @@ def parseIntegratedTestScript(test, additional_parsers=[],
return lit.Test.Result(Test.UNRESOLVED,
"Test has unterminated run lines (with '\\')")

# Check that we have the required features:
missing_required_features = [f for f in requires
if f not in test.config.available_features]
# Enforce REQUIRES:
missing_required_features = test.getMissingRequiredFeatures()
if missing_required_features:
msg = ', '.join(missing_required_features)
return lit.Test.Result(Test.UNSUPPORTED,
"Test requires the following features: %s"
% msg)
requires_any_features = [f for f in requires_any
if f in test.config.available_features]
if requires_any and not requires_any_features:
msg = ' ,'.join(requires_any)
return lit.Test.Result(Test.UNSUPPORTED,
"Test requires any of the following features: "
"%s" % msg)
unsupported_features = [f for f in unsupported
if f in test.config.available_features]
"Test requires the following unavailable "
"features: %s" % msg)

# Enforce UNSUPPORTED:
unsupported_features = test.getUnsupportedFeatures()
if unsupported_features:
msg = ', '.join(unsupported_features)
return lit.Test.Result(
Test.UNSUPPORTED,
"Test is unsupported with the following features: %s" % msg)
"Test does not support the following features "
"and/or targets: %s" % msg)

# Enforce limit_to_features.
if not test.isWithinFeatureLimits():
msg = ', '.join(test.config.limit_to_features)
return lit.Test.Result(Test.UNSUPPORTED,
"Test does not require any of the features "
"specified in limit_to_features: %s" % msg)

unsupported_targets = [f for f in unsupported
if f in test.suite.config.target_triple]
if unsupported_targets:
return lit.Test.Result(
Test.UNSUPPORTED,
"Test is unsupported with the following triple: %s" % (
test.suite.config.target_triple,))

if test.config.limit_to_features:
# Check that we have one of the limit_to_features features in requires.
limit_to_features_tests = [f for f in test.config.limit_to_features
if f in requires]
if not limit_to_features_tests:
msg = ', '.join(test.config.limit_to_features)
return lit.Test.Result(
Test.UNSUPPORTED,
"Test requires one of the limit_to_features features %s" % msg)
return script


Expand Down

This file was deleted.

This file was deleted.

3 changes: 3 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-format/requires-any.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# REQUIRES-ANY is no longer supported. Test should not run.
REQUIRES-ANY: true, a-present-feature
RUN: false
Original file line number Diff line number Diff line change
@@ -1,2 +1,5 @@
RUN: true
REQUIRES: a-missing-feature
# REQUIRES with a false clause. Test should not run.
REQUIRES: true
REQUIRES: a-missing-feature, true
REQUIRES: true
RUN: false
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
# REQUIRES with only true clauses. Test should run.
REQUIRES: a-present-feature, true, !not-true
REQUIRES: true
RUN: true
REQUIRES: a-present-feature
3 changes: 3 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-format/requires-star.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# '*' only works in XFAIL
REQUIRES: *
RUN: false
3 changes: 3 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-format/requires-triple.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# REQUIRES line that uses target triple, which doesn't work. Test should not run
REQUIRES: x86_64
RUN: false
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# UNSUPPORTED with only false clauses. Test should run.
UNSUPPORTED: false
UNSUPPORTED: false, not-true
UNSUPPORTED: false
UNSUPPORTED: still-not-true
UNSUPPORTED: false
UNSUPPORTED: false
UNSUPPORTED: false
RUN: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# UNSUPPORTED with a true clause. Test should not run.
UNSUPPORTED: false
UNSUPPORTED: false, false, false, _64-unk && a-present-feature, false
RUN: false
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# '*' only works in XFAIL
UNSUPPORTED: *
RUN: false
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# XFAIL with only false clauses. Test should run.
XFAIL: false, a-missing-feature || ! a-present-feature || ! x86_64, false
RUN: true
4 changes: 4 additions & 0 deletions llvm/utils/lit/tests/Inputs/shtest-format/xfail-expr-true.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# XFAIL with a true clause. Test should not run.
XFAIL: false
XFAIL: false, a-present-feature && ! a-missing-feature && x86_64
RUN: false
4 changes: 4 additions & 0 deletions llvm/utils/lit/tests/boolean-parsing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Test the boolean expression parser
# used for REQUIRES and UNSUPPORTED and XFAIL

# RUN: %{python} -m lit.BooleanExpression
19 changes: 13 additions & 6 deletions llvm/utils/lit/tests/shtest-format.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,18 @@

# CHECK: UNRESOLVED: shtest-format :: no-test-line.txt
# CHECK: PASS: shtest-format :: pass.txt
# CHECK: UNSUPPORTED: shtest-format :: requires-any-missing.txt
# CHECK: PASS: shtest-format :: requires-any-present.txt
# CHECK: UNRESOLVED: shtest-format :: requires-any.txt
# CHECK: ValueError: `REQUIRES-ANY: a, b, c` not supported
# CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt
# CHECK: PASS: shtest-format :: requires-present.txt
# CHECK: UNRESOLVED: shtest-format :: requires-star.txt
# CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt
# CHECK: PASS: shtest-format :: unsupported-expr-false.txt
# CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt
# CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt
# CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt
# CHECK: PASS: shtest-format :: xfail-expr-false.txt
# CHECK: XFAIL: shtest-format :: xfail-expr-true.txt
# CHECK: XFAIL: shtest-format :: xfail-feature.txt
# CHECK: XFAIL: shtest-format :: xfail-target.txt
# CHECK: XFAIL: shtest-format :: xfail.txt
Expand All @@ -70,9 +77,9 @@
# CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
# CHECK: shtest-format :: fail.txt

# CHECK: Expected Passes : 5
# CHECK: Expected Failures : 3
# CHECK: Unsupported Tests : 3
# CHECK: Unresolved Tests : 1
# CHECK: Expected Passes : 6
# CHECK: Expected Failures : 4
# CHECK: Unsupported Tests : 4
# CHECK: Unresolved Tests : 4
# CHECK: Unexpected Passes : 1
# CHECK: Unexpected Failures: 3
65 changes: 65 additions & 0 deletions llvm/utils/lit/tests/unit/TestRunner.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,71 @@ def test_custom(self):
value = custom_parser.getValue()
self.assertItemsEqual(value, ['a', 'b', 'c'])

def test_bad_keywords(self):
def custom_parse(line_number, line, output):
return output

try:
IntegratedTestKeywordParser("TAG_NO_SUFFIX", ParserKind.TAG),
self.fail("TAG_NO_SUFFIX failed to raise an exception")
except ValueError as e:
pass
except BaseException as e:
self.fail("TAG_NO_SUFFIX raised the wrong exception: %r" % e)

try:
IntegratedTestKeywordParser("TAG_WITH_COLON:", ParserKind.TAG),
self.fail("TAG_WITH_COLON: failed to raise an exception")
except ValueError as e:
pass
except BaseException as e:
self.fail("TAG_WITH_COLON: raised the wrong exception: %r" % e)

try:
IntegratedTestKeywordParser("LIST_WITH_DOT.", ParserKind.LIST),
self.fail("LIST_WITH_DOT. failed to raise an exception")
except ValueError as e:
pass
except BaseException as e:
self.fail("LIST_WITH_DOT. raised the wrong exception: %r" % e)

try:
IntegratedTestKeywordParser("CUSTOM_NO_SUFFIX",
ParserKind.CUSTOM, custom_parse),
self.fail("CUSTOM_NO_SUFFIX failed to raise an exception")
except ValueError as e:
pass
except BaseException as e:
self.fail("CUSTOM_NO_SUFFIX raised the wrong exception: %r" % e)

# Both '.' and ':' are allowed for CUSTOM keywords.
try:
IntegratedTestKeywordParser("CUSTOM_WITH_DOT.",
ParserKind.CUSTOM, custom_parse),
except BaseException as e:
self.fail("CUSTOM_WITH_DOT. raised an exception: %r" % e)
try:
IntegratedTestKeywordParser("CUSTOM_WITH_COLON:",
ParserKind.CUSTOM, custom_parse),
except BaseException as e:
self.fail("CUSTOM_WITH_COLON: raised an exception: %r" % e)

try:
IntegratedTestKeywordParser("CUSTOM_NO_PARSER:",
ParserKind.CUSTOM),
self.fail("CUSTOM_NO_PARSER: failed to raise an exception")
except ValueError as e:
pass
except BaseException as e:
self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e)

# REQUIRES-ANY: has a built-in parser that generates an error,
# but it may be overridden by a custom parser.
try:
IntegratedTestKeywordParser("REQUIRES-ANY:",
ParserKind.CUSTOM, custom_parse),
except BaseException as e:
self.fail("REQUIRES-ANY: raised an exception: %r" % e)

if __name__ == '__main__':
TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests()
Expand Down