Compile the LLDB tests out-of-tree.

This patch is the result of a discussion on lldb-dev, see
http://lists.llvm.org/pipermail/lldb-dev/2018-January/013111.html for
background.

For each test (should be eventually: each test configuration) a
separate build directory is created and we execute

  make VPATH=$srcdir/path/to/test -C $builddir/path/to/test -f $srcdir/path/to/test/Makefile -I $srcdir/path/to/test

In order to make this work all LLDB tests need to be updated to find
the executable in the test build directory, since CWD still points at
the test's source directory, which is a requirement for unittest2.

Although we have done extensive testing, I'm expecting that this first
attempt will break a few bots. Please DO NOT HESITATE TO REVERT this
patch in order to get the bots green again. We will likely have to
iterate on this some more.

Differential Revision: https://reviews.llvm.org/D42281

llvm-svn: 323803

lldb/docs/testsuite/a-detailed-walkthrough.txt

@@ -135,15 +135,16 @@ see test/array_types/TestArrayTypes.py:
         self.array_types()
 
 This method is decorated with a skipUnless decorator so that it will only gets
-included into the test suite if the platform it is running on is 'darwin', aka
-Mac OS X.
+included into the test suite if the platform it is running on is 'darwin', a.k.a.
+macOS.
 
 Type 'man dsymutil' for more details. 
 
 After the binary is built, it is time to specify the file to be used as the main
 executable by lldb:
 
-        exe = os.path.join(os.getcwd(), "a.out")
+        # Construct the path to a file "a.out" inside the test's build folder.
+        exe = self.getBuildArtifact("a.out")
         self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
 
 This is where the attribute assignment:
@@ -174,10 +175,11 @@ execution.  This can be accomplished by passing the keyword argument pair
 After the current executable is set, we'll then execute two more commands:
 
         # Set the output-path and verify it is set.
-        self.runCmd("settings set target.process.output-path 'stdout.txt'")
+        stdout = self.getBuildArtifact('stdout.txt')
+        self.runCmd("settings set target.process.output-path '%s'" %stdout)
         self.expect("settings show target.process.output-path",
                     SETTING_MSG("target.process.output-path"),
-            startstr = "target.process.output-path (string) = 'stdout.txt'")
+            startstr = "target.process.output-path (string) = '.*stdout.txt'")
 
 The first uses the 'settings set' command to set the static setting
 target.process.output-path to be 'stdout.txt', instead of the default
@@ -188,7 +190,7 @@ door and grabs the output from the command execution and expects to match the
 start string of the output against what we pass in as the value of the keyword
 argument pair:
 
-            startstr = "target.process.output-path (string) = 'stdout.txt'"
+            startstr = "target.process.output-path (string) = '%s'" %stdout
 
 Take a look at TestBase.expect() within lldbtest.py for more details.  Among
 other things, it can also match against a list of regexp patterns as well as a
@@ -204,15 +206,15 @@ And this asserts that the file 'stdout.txt' should be present after running the
 program.
 
         # The 'stdout.txt' file should now exist.
-        self.assertTrue(os.path.isfile("stdout.txt"),
-                        "'stdout.txt' exists due to target.process.output-path.")
+        self.assertTrue(os.path.isfile(stdout),
+                        "stdout.txt' exists due to target.process.output-path.")
 
 Also take a look at main.cpp which emits some message to the stdout.  Now, if we
 pass this assertion, it's time to examine the contents of the file to make sure
 it contains the same message as programmed in main.cpp:
 
         # Read the output file produced by running the program.
-        with open('stdout.txt', 'r') as f:
+        with open(stdout, 'r') as f:
             output = f.read()
 
         self.expect(output, exe=False,
@@ -235,8 +237,8 @@ file:
 
     @classmethod
     def classCleanup(cls):
-        system(["/bin/sh", "-c", "rm -f output.txt"])
-        system(["/bin/sh", "-c", "rm -f stdout.txt"])
+        system(["/bin/sh", "-c", "rm -f "+self.getBuildArtifact("output.txt")])
+        system(["/bin/sh", "-c", "rm -f "+self.getBuildArtifact("stdout.txt")])
 
 This is a classmethod (as shown by the @classmethod decorator) which allows the
 individual test class to perform cleanup actions after the test harness finishes

lldb/packages/Python/lldbsuite/test/api/check_public_api_headers/TestPublicAPIHeaders.py

@@ -21,7 +21,6 @@ class SBDirCheckerCase(TestBase):
     def setUp(self):
         TestBase.setUp(self)
         self.source = 'main.cpp'
-        self.exe_name = self.getBuildArtifact("a.out")
         self.generateSource(self.source)
 
     @skipIfNoSBHeaders
@@ -35,16 +34,19 @@ def test_sb_api_directory(self):
             self.skipTest(
                 "LLDB is 64-bit and cannot be linked to 32-bit test program.")
 
-        self.buildDriver(self.source, self.exe_name)
-        self.sanity_check_executable(self.exe_name)
+        exe_name = self.getBuildArtifact("a.out")
+        self.buildDriver(self.source, exe_name)
+        self.sanity_check_executable(exe_name)
 
     def sanity_check_executable(self, exe_name):
         """Sanity check executable compiled from the auto-generated program."""
+        exe_name = self.getBuildArtifact("a.out")
         exe = self.getBuildArtifact(exe_name)
         self.runCmd("file %s" % exe, CURRENT_EXECUTABLE_SET)
 
+        # This test uses a generated source file, so it's in the build directory.
         self.line_to_break = line_number(
-            self.source, '// Set breakpoint here.')
+            self.getBuildArtifact(self.source), '// Set breakpoint here.')
 
         env_cmd = "settings set target.env-vars %s=%s" % (
             self.dylibPath, self.getLLDBLibraryEnvVal())

lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/TestMultipleDebuggers.py

@@ -32,12 +32,12 @@ class TestMultipleSimultaneousDebuggers(TestBase):
     def test_multiple_debuggers(self):
         env = {self.dylibPath: self.getLLDBLibraryEnvVal()}
 
-        self.driver_exe = os.path.join(os.getcwd(), "multi-process-driver")
+        self.driver_exe = self.getBuildArtifact("multi-process-driver")
         self.buildDriver('multi-process-driver.cpp', self.driver_exe)
         self.addTearDownHook(lambda: os.remove(self.driver_exe))
         self.signBinary(self.driver_exe)
 
-        self.inferior_exe = os.path.join(os.getcwd(), "testprog")
+        self.inferior_exe = self.getBuildArtifact("testprog")
         self.buildDriver('testprog.cpp', self.inferior_exe)
         self.addTearDownHook(lambda: os.remove(self.inferior_exe))
 

lldb/packages/Python/lldbsuite/test/api/multiple-targets/TestMultipleTargets.py

@@ -26,7 +26,7 @@ class TestMultipleTargets(TestBase):
     def test_multiple_targets(self):
         env = {self.dylibPath: self.getLLDBLibraryEnvVal()}
 
-        self.driver_exe = os.path.join(os.getcwd(), "multi-target")
+        self.driver_exe = self.getBuildArtifact("multi-target")
         self.buildDriver('main.cpp', self.driver_exe)
         self.addTearDownHook(lambda: os.remove(self.driver_exe))
         self.signBinary(self.driver_exe)

lldb/packages/Python/lldbsuite/test/api/multithreaded/TestMultithreaded.py

@@ -89,14 +89,16 @@ def build_and_test(self, sources, test_name, args=None):
 
         self.inferior = 'inferior_program'
         self.buildProgram('inferior.cpp', self.inferior)
-        self.addTearDownHook(lambda: os.remove(self.inferior))
+        self.addTearDownHook(lambda:
+                             os.remove(self.getBuildArtifact(self.inferior)))
 
         self.buildDriver(sources, test_name)
-        self.addTearDownHook(lambda: os.remove(test_name))
+        self.addTearDownHook(lambda:
+                             os.remove(self.getBuildArtifact(test_name)))
 
-        test_exe = os.path.join(os.getcwd(), test_name)
+        test_exe = self.getBuildArtifact(test_name)
         self.signBinary(test_exe)
-        exe = [test_exe, self.inferior]
+        exe = [test_exe, self.getBuildArtifact(self.inferior)]
 
         env = {self.dylibPath: self.getLLDBLibraryEnvVal()}
         if self.TraceOn():

lldb/packages/Python/lldbsuite/test/arm/emulation/TestEmulations.py

@@ -19,8 +19,7 @@ class ARMEmulationTestCase(TestBase):
 
     @no_debug_info_test
     def test_thumb_emulations(self):
-        current_dir = os.getcwd()
-        test_dir = os.path.join(current_dir, "new-test-files")
+        test_dir = os.path.join(self.getSourceDir(), "new-test-files")
         files = os.listdir(test_dir)
         thumb_files = list()
         for f in files:
@@ -33,8 +32,7 @@ def test_thumb_emulations(self):
 
     @no_debug_info_test
     def test_arm_emulations(self):
-        current_dir = os.getcwd()
-        test_dir = os.path.join(current_dir, "new-test-files")
+        test_dir = os.path.join(self.getSourceDir(), "new-test-files")
         files = os.listdir(test_dir)
         arm_files = list()
         for f in files:

lldb/packages/Python/lldbsuite/test/benchmarks/expression/TestRepeatedExprs.py

@@ -94,7 +94,7 @@ def run_lldb_repeated_exprs(self, exe_name, count):
 
     def run_gdb_repeated_exprs(self, exe_name, count):
         import pexpect
-        exe = os.path.join(os.getcwd(), exe_name)
+        exe = self.getBuildArtifact(exe_name)
 
         # Set self.child_prompt, which is "(gdb) ".
         self.child_prompt = '(gdb) '

lldb/packages/Python/lldbsuite/test/configuration.py

@@ -107,6 +107,9 @@
 lldb_platform_url = None
 lldb_platform_working_dir = None
 
+# The base directory in which the tests are being built.
+test_build_dir = None
+
 # Parallel execution settings
 is_inferior_test_runner = False
 multiprocess_test_subdir = None

lldb/packages/Python/lldbsuite/test/darwin_log.py

@@ -162,7 +162,7 @@ def do_test(self, enable_options, expect_regexes=None,
         if enable_options is not None and len(enable_options) > 0:
             enable_cmd += ' ' + ' '.join(enable_options)
 
-        exe = os.path.join(os.getcwd(), self.exe_name)
+        exe = self.getBuildArtifact(self.exe_name)
         self.run_lldb_to_breakpoint(exe, self.source, self.line,
                                     enable_command=enable_cmd,
                                     settings_commands=settings_commands)
@@ -382,7 +382,7 @@ def do_test(self, enable_options, settings_commands=None,
         # self.runCmd("log enable lldb process")
 
         # Launch the process - doesn't stop at entry.
-        process = target.LaunchSimple(None, None, os.getcwd())
+        process = target.LaunchSimple(None, None, self.getBuildDir())
         self.assertIsNotNone(process, lldbtest.PROCESS_IS_VALID)
 
         # Keep track of whether we're tracing output.

lldb/packages/Python/lldbsuite/test/dotest.py

@@ -474,6 +474,8 @@ def parseOptionsAndInitTestdirs():
         configuration.lldb_platform_url = args.lldb_platform_url
     if args.lldb_platform_working_dir:
         configuration.lldb_platform_working_dir = args.lldb_platform_working_dir
+    if args.test_build_dir:
+        configuration.test_build_dir = args.test_build_dir
 
     if args.event_add_entries and len(args.event_add_entries) > 0:
         entries = {}
@@ -623,6 +625,12 @@ def setupSysPath():
 
     os.environ["LLDB_TEST"] = scriptPath
 
+    # Set up the root build directory.
+    builddir = configuration.test_build_dir
+    if not configuration.test_build_dir:
+        raise Exception("test_build_dir is not set")
+    os.environ["LLDB_BUILD"] = os.path.abspath(configuration.test_build_dir)
+
     # Set up the LLDB_SRC environment variable, so that the tests can locate
     # the LLDB source code.
     os.environ["LLDB_SRC"] = lldbsuite.lldb_root
@@ -1186,6 +1194,11 @@ def run_suite():
         configuration.lldb_platform_working_dir = None
         configuration.lldb_platform_url = None
 
+    # Set up the working directory.
+    # Note that it's not dotest's job to clean this directory.
+    try: os.makedirs(configuration.test_build_dir)
+    except: pass
+
     target_platform = lldb.DBG.GetSelectedPlatform().GetTriple().split('-')[2]
 
     checkLibcxxSupport()

lldb/packages/Python/lldbsuite/test/dotest_args.py

@@ -159,6 +159,12 @@ def create_parser():
         metavar='Codesigning identity',
         default='lldb_codesign',
         help='The codesigning identity to use')
+    group.add_argument(
+        '--build-dir',
+        dest='test_build_dir',
+        metavar='Test build directory',
+        default='lldb-test-build',
+        help='The root build directory for the tests. It will be removed before running.')
 
     # Configuration options
     group = parser.add_argument_group('Remote platform options')

lldb/packages/Python/lldbsuite/test/expression_command/call-overridden-method/TestCallOverriddenMethod.py

@@ -31,7 +31,8 @@ def test(self):
         self.build()
 
         # Set breakpoint in main and run exe
-        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)
+        self.runCmd("file " + self.getBuildArtifact("a.out"),
+                    CURRENT_EXECUTABLE_SET)
         lldbutil.run_break_set_by_file_and_line(
             self, "main.cpp", self.line, num_expected_locations=-1, loc_exact=True)
 

lldb/packages/Python/lldbsuite/test/expression_command/save_jit_objects/TestSaveJITObjects.py

@@ -11,23 +11,24 @@
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-def enumerateJITFiles():
-    return [f for f in os.listdir(os.getcwd()) if f.startswith("jit")]
-
-def countJITFiles():
-    return len(enumerateJITFiles())
-
-def cleanJITFiles():
-    for j in enumerateJITFiles():
-        os.remove(j)
-    return
-
 class SaveJITObjectsTestCase(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
+    def enumerateJITFiles(self):
+        return [f for f in os.listdir(self.getBuildDir()) if f.startswith("jit")]
+
+    def countJITFiles(self):
+        return len(self.enumerateJITFiles())
+
+    def cleanJITFiles(self):
+        for j in self.enumerateJITFiles():
+            os.remove(j)
+        return
+
     @expectedFailureAll(oslist=["windows"])
     def test_save_jit_objects(self):
         self.build()
+        os.chdir(self.getBuildDir())
         src_file = "main.c"
         src_file_spec = lldb.SBFileSpec(src_file)
 
@@ -36,16 +37,17 @@ def test_save_jit_objects(self):
 
         frame = thread.frames[0]
 
-        cleanJITFiles()
+        self.cleanJITFiles()
         frame.EvaluateExpression("(void*)malloc(0x1)")
-        self.assertTrue(countJITFiles() == 0,
+        self.assertTrue(self.countJITFiles() == 0,
                         "No files emitted with save-jit-objects=false")
 
         self.runCmd("settings set target.save-jit-objects true")
         frame.EvaluateExpression("(void*)malloc(0x1)")
-        jit_files_count = countJITFiles()
-        cleanJITFiles()
+        jit_files_count = self.countJITFiles()
+        self.cleanJITFiles()
         self.assertTrue(jit_files_count != 0,
                         "At least one file emitted with save-jit-objects=true")
 
         process.Kill()
+        os.chdir(self.getSourceDir())

lldb/packages/Python/lldbsuite/test/expression_command/top-level/Makefile

@@ -7,7 +7,7 @@ include $(LEVEL)/Makefile.rules
 a.out: dummy
 
 dummy:
-	$(MAKE) -f dummy.mk
+	$(MAKE) VPATH=$(VPATH) -I $(SRCDIR) -f $(SRCDIR)/dummy.mk
 
 clean::
-	$(MAKE) -f dummy.mk clean
+	$(MAKE) VPATH=$(VPATH) -I $(SRCDIR) -f $(SRCDIR)/dummy.mk clean

lldb/packages/Python/lldbsuite/test/expression_command/top-level/TestTopLevelExprs.py

@@ -45,7 +45,8 @@ def build_and_run(self):
         self.runCmd("run", RUN_SUCCEEDED)
 
     def run_dummy(self):
-        self.runCmd("file dummy", CURRENT_EXECUTABLE_SET)
+        self.runCmd("file " + self.getBuildArtifact("dummy"),
+                    CURRENT_EXECUTABLE_SET)
 
         lldbutil.run_break_set_by_file_and_line(
             self,

lldb/packages/Python/lldbsuite/test/functionalities/attach_resume/TestAttachResume.py

@@ -30,7 +30,7 @@ def test_attach_continue_interrupt_detach(self):
     def process_attach_continue_interrupt_detach(self):
         """Test attach/continue/interrupt/detach"""
 
-        exe = os.path.join(os.getcwd(), exe_name)
+        exe = self.getBuildArtifact(exe_name)
 
         popen = self.spawnSubprocess(exe)
         self.addTearDownHook(self.cleanupSubprocesses)

lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/TestBadAddressBreakpoints.py

@@ -30,9 +30,8 @@ def setUp(self):
 
     def address_breakpoints(self):
         """Test that breakpoints set on a bad address say they are bad."""
-
-        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self, 
-                "Set a breakpoint here", lldb.SBFileSpec("main.c"))
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "Set a breakpoint here", lldb.SBFileSpec("main.c"))
 
         # Now see if we can read from 0.  If I can't do that, I don't have a good way to know
         # what an illegal address is...

lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_case_sensitivity/TestBreakpointCaseSensitivity.py

@@ -44,24 +44,24 @@ def case_sensitivity_breakpoint(self, case_insensitive):
         # Create a target by the debugger.
         self.target = self.dbg.CreateTarget(exe)
         self.assertTrue(self.target, VALID_TARGET)
-        cwd = os.getcwd()
+        srcdir = self.getSourceDir()
 
         # try both BreakpointCreateByLocation and BreakpointCreateBySourceRegex
         for regex in [False, True]:
             # should always hit
             self.check_breakpoint('main.c', regex, True)
             # should always hit
-            self.check_breakpoint(os.path.join(cwd, 'main.c'), regex, True)
+            self.check_breakpoint(os.path.join(srcdir, 'main.c'), regex, True)
             # different case for directory
-            self.check_breakpoint(os.path.join(cwd.upper(), 'main.c'),
+            self.check_breakpoint(os.path.join(srcdir.upper(), 'main.c'),
                                   regex,
                                   case_insensitive)
             # different case for file
             self.check_breakpoint('Main.c',
                                   regex,
                                   case_insensitive)
             # different case for both
-            self.check_breakpoint(os.path.join(cwd.upper(), 'Main.c'),
+            self.check_breakpoint(os.path.join(srcdir.upper(), 'Main.c'),
                                   regex,
                                   case_insensitive)
 

lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/TestRegexpBreakCommand.py

@@ -54,7 +54,7 @@ def regexp_break_command(self):
             num_locations=1)
 
         # Check breakpoint with full file path.
-        full_path = os.path.join(os.getcwd(), self.source)
+        full_path = os.path.join(self.getSourceDir(), self.source)
         break_results = lldbutil.run_break_set_command(
             self, "b %s:%d" % (full_path, self.line))
         lldbutil.check_breakpoint_result(