[lit] Fix shell commands with newlines (#67898)

In PR #65242 (landed as 9e739fd), I
claimed that RUN lines cannot contain newlines. Actually, they can after
substitution expansion. More generally, a lit config file can define
substitutions or preamble commands containing newlines. While both of
those cases seem unlikely in practice,
[D154987](https://reviews.llvm.org/D154987) proposes PYTHON directives
where it seems very likely.

Regardless of the use case, without this patch, such newlines break
expansion of `%dbg(RUN: at line N)`, and the fix is simple.

llvm/utils/lit/lit/TestRunner.py

@@ -55,7 +55,7 @@ def __init__(self, command, message):
 #
 # COMMAND that follows %dbg(ARG) is also captured. COMMAND can be
 # empty as a result of conditinal substitution.
-kPdbgRegex = "%dbg\\(([^)'\"]*)\\)(.*)"
+kPdbgRegex = "%dbg\\(([^)'\"]*)\\)((?:.|\\n)*)"
 
 
 def buildPdbgCommand(msg, cmd):

llvm/utils/lit/tests/Inputs/shtest-inject/lit.cfg

@@ -1,6 +1,7 @@
 import lit
 
-preamble_commands = ['echo "THIS WAS"', 'echo "INJECTED"']
+# Check multiple commands, and check newlines.
+preamble_commands = ['echo "THIS WAS"', 'echo\n"INJECTED"']
 
 config.name = "shtest-inject"
 config.suffixes = [".txt"]

llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg

@@ -1,3 +1,8 @@
 import lit.formats
 
 config.test_format = lit.formats.ShTest(execute_external=True)
+config.substitutions.append(("%{cmds-with-newlines}", """
+echo abc |
+FileCheck %s &&
+false
+"""))

llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/run-line-with-newline.txt

@@ -0,0 +1,2 @@
+# RUN: %{cmds-with-newlines}
+# CHECK: abc

llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg

@@ -1,3 +1,8 @@
 import lit.formats
 
 config.test_format = lit.formats.ShTest(execute_external=False)
+config.substitutions.append(("%{cmds-with-newlines}", """
+echo abc |
+FileCheck %s &&
+false
+"""))

llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/run-line-with-newline.txt

@@ -0,0 +1,2 @@
+# RUN: %{cmds-with-newlines}
+# CHECK: abc

llvm/utils/lit/tests/shtest-inject.py

@@ -14,7 +14,8 @@
 #  CHECK-TEST1-NEXT: # | THIS WAS
 #  CHECK-TEST1-NEXT: # `---{{-*}}
 #  CHECK-TEST1-NEXT: # preamble command line
-#  CHECK-TEST1-NEXT: echo "INJECTED"
+#  CHECK-TEST1-NEXT: echo
+#  CHECK-TEST1-NEXT: "INJECTED"
 #  CHECK-TEST1-NEXT: # executed command: echo INJECTED
 #  CHECK-TEST1-NEXT: # .---command stdout{{-*}}
 #  CHECK-TEST1-NEXT: # | INJECTED

llvm/utils/lit/tests/shtest-run-at-line.py

@@ -7,7 +7,7 @@
 # END.
 
 
-# CHECK: Testing: 6 tests
+# CHECK: Testing: 8 tests
 
 
 # In the case of the external shell, we check for only RUN lines in stderr in
@@ -48,6 +48,15 @@
 #   CHECK-NOT: RUN
 #       CHECK: --
 
+# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/run-line-with-newline.txt
+
+#      CHECK: Command Output (stderr)
+# CHECK-NEXT: --
+# CHECK-NEXT: {{^}}RUN: at line 1: echo abc |
+# CHECK-NEXT: FileCheck {{.*}} &&
+# CHECK-NEXT: false
+#  CHECK-NOT: RUN
+
 
 # CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt
 
@@ -87,3 +96,16 @@
 # CHECK-NEXT: # executed command: echo 'foo baz'
 # CHECK-NEXT: # executed command: FileCheck {{.*}}
 # CHECK-NOT:  RUN
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/run-line-with-newline.txt
+
+#      CHECK: Command Output (stdout)
+# CHECK-NEXT: --
+# CHECK-NEXT: # RUN: at line 1
+# CHECK-NEXT: echo abc |
+# CHECK-NEXT: FileCheck {{.*}} &&
+# CHECK-NEXT: false
+# CHECK-NEXT: # executed command: echo abc
+# CHECK-NEXT: # executed command: FileCheck {{.*}}
+# CHECK-NEXT: # executed command: false
+#  CHECK-NOT: RUN