diff --git a/docs/07-known-issues.rst b/docs/07-known-issues.rst new file mode 100644 index 0000000..9bd6435 --- /dev/null +++ b/docs/07-known-issues.rst @@ -0,0 +1,38 @@ +Known issues +============ + +Unintuitive behavior of CHECK-NOT +--------------------------------- + +A failing CHECK has a higher precedence than a failing CHECK-NOT +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A failing `CHECK` has a higher precedence than a failing `CHECK-NOT` even if +`CHECK-NOT` appears first in the check file. If this happens, the output is +related to the failing `CHECK`, not the failing `CHECK-NOT`. + +Input: + +.. code-block:: text + + String1 + Stringggg2 + +Check file: + +.. code-block:: text + + ; CHECK-NOT:String1 + ; CHECK:String2 + +Result: + +.. code-block:: text + + /Users/Stanislaw/.pyenv/shims/filecheck + filecheck.check:2:9: error: CHECK: expected string not found in input + ; CHECK:String2 + ^ + :1:1: note: scanning from here + String1 + ^ diff --git a/docs/index.rst b/docs/index.rst index 88633b9..ba44b2e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,6 +10,7 @@ Welcome to FileCheck.py's documentation! 04-tutorial-llvm-lit-and-filecheck 05-check-commands 06-options + 07-known-issues .. Indices and tables ================== diff --git a/filecheck/FileCheck.py b/filecheck/FileCheck.py index 8be9b7d..c3be6e8 100755 --- a/filecheck/FileCheck.py +++ b/filecheck/FileCheck.py @@ -12,8 +12,20 @@ __version__ = '0.0.5' -class CheckFailedException(Exception): - pass +class FailedCheck: + def __init__(self, check, line_idx): + self.check = check + self.line_idx = line_idx + + +class CheckFailedException(BaseException): + def __init__(self, failed_check): + self.failed_check = failed_check + + +class InputFinishedException(BaseException): + def __init__(self): + pass class MatchType(Enum): @@ -105,7 +117,8 @@ class CheckResult(Enum): PASS = 1 FAIL_SKIP_LINE = 2 FAIL_FATAL = 3 - CHECK_NOT_WITHOUT_MATCH = 4 + CHECK_NOT_MATCH = 4 + CHECK_NOT_WITHOUT_MATCH = 5 def check_line(line, current_check, match_full_lines): @@ -142,13 +155,13 @@ def check_line(line, current_check, match_full_lines): elif current_check.check_type == CheckType.CHECK_NOT: if current_check.match_type == MatchType.SUBSTRING: if current_check.expression in line: - return CheckResult.FAIL_FATAL + return CheckResult.CHECK_NOT_MATCH else: return CheckResult.CHECK_NOT_WITHOUT_MATCH elif current_check.match_type == MatchType.REGEX: if re.search(current_check.expression, line): - return CheckResult.FAIL_FATAL + return CheckResult.CHECK_NOT_MATCH else: return CheckResult.CHECK_NOT_WITHOUT_MATCH @@ -246,7 +259,7 @@ def main(): checks.append(check) continue - check_not_regex = "; {}-NOT: (.*)".format(check_prefix) + check_not_regex = "; {}-NOT:{}(.*)".format(check_prefix, strict_whitespace_match) check_match = re.search(check_not_regex, line) if check_match: match_type = MatchType.SUBSTRING @@ -291,6 +304,11 @@ def main(): check_iterator = iter(checks) current_check = None + # This variable is currently only used for CHECK-NOT checks which do not + # necessarily fail with a last input line. So we have to keep the failing + # line index. + current_check_line_idx = None + try: current_check = next(check_iterator) except StopIteration: @@ -315,6 +333,9 @@ def main(): exit(2) try: + current_not_checks = [] + failed_check = None + while True: line = line.rstrip() if not args.strict_whitespace: @@ -323,9 +344,25 @@ def main(): input_lines.append(line) while True: + if not failed_check: + for current_not_check in current_not_checks: + check_result = check_line(line, + current_not_check, + args.match_full_lines) + if check_result == CheckResult.CHECK_NOT_MATCH: + failed_check = FailedCheck(current_not_check, line_idx) + check_result = check_line(line, current_check, args.match_full_lines) - if check_result == CheckResult.PASS: + if check_result == CheckResult.FAIL_FATAL: + failed_check = FailedCheck(current_check, line_idx) + raise CheckFailedException(failed_check) + + elif check_result == CheckResult.PASS: + if failed_check: + raise CheckFailedException(failed_check) + + current_not_checks.clear() try: current_check = next(check_iterator) except StopIteration: @@ -336,27 +373,54 @@ def main(): current_scan_base = line_idx break except StopIteration: - raise CheckFailedException() + raise InputFinishedException + + elif check_result == CheckResult.CHECK_NOT_MATCH: + failed_check = FailedCheck(current_check, line_idx) + try: + current_not_checks.append(current_check) + current_check = next(check_iterator) + continue + except StopIteration: + raise CheckFailedException(failed_check) elif check_result == CheckResult.CHECK_NOT_WITHOUT_MATCH: try: + current_not_checks.append(current_check) current_check = next(check_iterator) + continue except StopIteration: exit(0) - elif check_result == CheckResult.FAIL_FATAL: - raise CheckFailedException() - elif check_result == CheckResult.FAIL_SKIP_LINE: try: line_idx, line = next(stdin_input_iter) break except StopIteration: - raise CheckFailedException() - else: - assert 0 - except CheckFailedException: - pass + failed_check = FailedCheck(current_check, line_idx) + raise CheckFailedException(failed_check) + + assert 0, "Should not reach here" + except InputFinishedException: + # We reach here if there is no input anymore and no check has failed so + # far. This means we can remove all CHECK-NOT checks from the input + # checks and see if there any other checks left. + if current_check.check_type == CheckType.CHECK_NOT: + still_actual_check = None + for check in check_iterator: + if check.check_type != CheckType.CHECK_NOT: + still_actual_check = check + break + else: + # No checks which are still actual have been found. Declare success. + exit(0) + + current_check = still_actual_check + current_check_line_idx = line_idx + + except CheckFailedException as e: + current_check = e.failed_check.check + current_check_line_idx = e.failed_check.line_idx # CHECK-EMPTY is special: if there is no output anymore and this check is # the 1) current and 2) the last one we want to declare success. @@ -436,7 +500,8 @@ def main(): if current_check.check_type == CheckType.CHECK_NOT: if (current_check.match_type == MatchType.SUBSTRING or current_check.match_type == MatchType.REGEX): - last_read_line = input_lines[-1] + assert current_check_line_idx != None + last_read_line = input_lines[current_check_line_idx] if not args.strict_whitespace: last_read_line = re.sub("\\s+", ' ', last_read_line).strip() @@ -448,15 +513,16 @@ def main(): print(current_check.source_line.rstrip()) print("^".rjust(current_check.start_index + 1)) - print(":{}:{}: note: found here".format(current_scan_base + 1, 1)) + print(":{}:{}: note: found here".format(current_check_line_idx + 1, 1)) print(last_read_line) if current_check.match_type == MatchType.SUBSTRING: match_pos = last_read_line.find(current_check.expression) assert match_pos != -1 + # TODO: check on lines which start with spaces highlight_line = "^".rjust(match_pos, ' ') - print("^".ljust(len(current_check.expression), '~')) + print(highlight_line.ljust(len(current_check.expression), '~')) else: print("^".ljust(len(last_read_line), '~')) diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/filecheck.check b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/filecheck.check new file mode 100644 index 0000000..794e6fe --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/filecheck.check @@ -0,0 +1,2 @@ +; CHECK-NOT:{{.*String1.*}} +; CHECK:String2 \ No newline at end of file diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/filecheck.input b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/filecheck.input new file mode 100644 index 0000000..56dde70 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/filecheck.input @@ -0,0 +1,3 @@ +NOT RELEVANT +String1 +String2 \ No newline at end of file diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/sample.itest b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/sample.itest new file mode 100644 index 0000000..0660ce1 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/01-failing-CHECK-NOT-followed-by-valid-CHECK/sample.itest @@ -0,0 +1,9 @@ +; RUN: cat %S/filecheck.input | (%FILECHECK_EXEC %S/filecheck.check 2>&1; test $? = 1;) | %FILECHECK_TESTER_EXEC %s --strict-whitespace --match-full-lines +; CHECK:{{^.*}}FileCheck{{(\.py)?$}} +; CHECK:{{^.*}}:1:13: error: CHECK-NOT: excluded string found in input +; CHECK:{{; CHECK-NOT:..\.\*String1.*..}} +; CHECK:{{^ \^$}} +; CHECK::2:1: note: found here +; CHECK:String1 +; CHECK:^~~~~~~ +; CHECK-EMPTY: diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/filecheck.check b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/filecheck.check new file mode 100644 index 0000000..19f25ed --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/filecheck.check @@ -0,0 +1,2 @@ +; CHECK:String2 +; CHECK-NOT:{{.*String1.*}} diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/filecheck.input b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/filecheck.input new file mode 100644 index 0000000..e3fe4f3 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/filecheck.input @@ -0,0 +1,2 @@ +String1 +String2 \ No newline at end of file diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/sample.itest b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/sample.itest new file mode 100644 index 0000000..42b48f9 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/03-CHECK-NOT-has-no-effect-if-next-CHECK-passed/sample.itest @@ -0,0 +1,3 @@ +; RUN: cat %S/filecheck.input | (%FILECHECK_EXEC %S/filecheck.check 2>&1; test $? = 0;) | %FILECHECK_TESTER_EXEC %s --strict-whitespace --match-full-lines +; CHECK:{{^.*}}FileCheck{{(\.py)?$}} +; CHECK-EMPTY: diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/filecheck.check b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/filecheck.check new file mode 100644 index 0000000..03b9112 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/filecheck.check @@ -0,0 +1,3 @@ +; CHECK:String2 +; CHECK-NOT:{{.*String1.*}} +; CHECK:String3 diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/filecheck.input b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/filecheck.input new file mode 100644 index 0000000..e3fe4f3 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/filecheck.input @@ -0,0 +1,2 @@ +String1 +String2 \ No newline at end of file diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/sample.itest b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/sample.itest new file mode 100644 index 0000000..04338c5 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/04-when-no-input-anymore-filters-out-CHECK-NOT-but-fails-on-CHECK/sample.itest @@ -0,0 +1,13 @@ +; RUN: cat %S/filecheck.input | (%FILECHECK_EXEC %S/filecheck.check 2>&1; test $? = 1;) | %FILECHECK_TESTER_EXEC %s --strict-whitespace --match-full-lines +; CHECK:{{^.*}}FileCheck{{(\.py)?$}} +; CHECK:{{^.*}}filecheck.check:3:9: error: CHECK: expected string not found in input{{$}} +; CHECK:; CHECK:String3 +; CHECK: ^ +; TODO: Without --match-full-lines, LLVM FileCheck allows multiple checks on a same line #52 +; TODO: https://github.com/stanislaw/FileCheck.py/issues/52 +; CHECK:{{:(1|2):(8|1): note: scanning from here}} +; CHECK:{{.*}} +; CHECK:{{.*^}} +; TODO: "note: possible intended match here" feature: not clear when FileCheck decides to show it or not #63 +; TODO: https://github.com/stanislaw/FileCheck.py/issues/63 +; CHECK-EMPTY diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/filecheck.check b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/filecheck.check new file mode 100644 index 0000000..2cc4a4b --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/filecheck.check @@ -0,0 +1,2 @@ +; CHECK-NOT:String1 +; CHECK:String2 \ No newline at end of file diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/filecheck.input b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/filecheck.input new file mode 100644 index 0000000..62a2abc --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/filecheck.input @@ -0,0 +1,2 @@ +String1 +Stringggg2 \ No newline at end of file diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/sample.itest b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/sample.itest new file mode 100644 index 0000000..d236d2a --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/01-minimal-example/sample.itest @@ -0,0 +1,11 @@ +; RUN: cat %S/filecheck.input | (%FILECHECK_EXEC %S/filecheck.check 2>&1; test $? = 1;) | %FILECHECK_TESTER_EXEC %s --strict-whitespace --match-full-lines +; CHECK:{{^.*}}FileCheck{{(\.py)?$}} +; CHECK:{{^.*}}filecheck.check:2:9: error: CHECK: expected string not found in input +; CHECK:; CHECK:String2 +; CHECK: ^ +; CHECK::1:1: note: scanning from here +; CHECK:String1 +; CHECK:^ +; TODO: Difference in behavior: FileCheck.py prints possible intended match, +; TODO: while FileCheck C++ does not. +; CHECK-EMPTY diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/filecheck.check b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/filecheck.check new file mode 100644 index 0000000..679ec0f --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/filecheck.check @@ -0,0 +1,2 @@ +; CHECK-NOT:{{.*String2.*}} +; CHECK:String3 \ No newline at end of file diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/filecheck.input b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/filecheck.input new file mode 100644 index 0000000..b0fd9c9 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/filecheck.input @@ -0,0 +1,3 @@ +String1 +String2 +Stringggg3 \ No newline at end of file diff --git a/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/sample.itest b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/sample.itest new file mode 100644 index 0000000..5371684 --- /dev/null +++ b/tests/integration/tests/check_commands/CHECK-NOT/multiple_strings/CHECK-has-a-higher-priority-than-CHECK-NOT/02-example-with-placeholder-string/sample.itest @@ -0,0 +1,11 @@ +; RUN: cat %S/filecheck.input | (%FILECHECK_EXEC %S/filecheck.check 2>&1; test $? = 1;) | %FILECHECK_TESTER_EXEC %s --strict-whitespace --match-full-lines +; CHECK:{{^.*}}FileCheck{{(\.py)?$}} +; CHECK:{{^.*}}:2:9: error: CHECK: expected string not found in input +; CHECK:; CHECK:String3 +; CHECK: ^ +; CHECK::1:1: note: scanning from here +; CHECK:String1 +; CHECK:^ +; TODO: Difference in behavior: FileCheck.py prints possible intended match, +; TODO: while FileCheck C++ does not. +; CHECK-EMPTY