[libcxx] Allow use of ShTest in libc++ tests along with other changes.

Summary:
This patch allows the use of LIT's ShTest format in the libc++ test suite. ShTests have the suffix '.sh.cpp'. It also introduces a series of other changes. These changes are:

- More functionality including parsing test metadata has been moved into LIT.
- LibcxxTestFormat now supports multi-part suffixes.
- the `CXXCompiler` functionality has been used to shrink the size of LibcxxTestFormat. 
- The recursive loading of the site config has been turned into `libcxx.test.config.loadSiteConfig` so it can be used with libc++abi.
- Temporary files are now created in the build directory of libc++. This follows how it is down in ShTest.
- `not.py` was added as a utility executable that mirrors the functionality of LLVM's `not` executable. 
- The first ShTest test was added under test/libcxx/double_include.sh.cpp


Reviewers: jroelofs, danalbert

Reviewed By: danalbert

Subscribers: cfe-commits

Differential Revision: http://reviews.llvm.org/D7073

llvm-svn: 226844

libcxx/test/libcxx/compiler.py

@@ -1,13 +1,15 @@
-
+import os
 import lit.util
+import libcxx.util
 
 
 class CXXCompiler(object):
-    def __init__(self, path, flags=[], compile_flags=[], link_flags=[], use_ccache=False):
+    def __init__(self, path, flags=None, compile_flags=None, link_flags=None,
+                 use_ccache=False):
         self.path = path
-        self.flags = list(flags)
-        self.compile_flags = list(compile_flags)
-        self.link_flags = list(link_flags)
+        self.flags = list(flags or [])
+        self.compile_flags = list(compile_flags or [])
+        self.link_flags = list(link_flags or [])
         self.use_ccache = use_ccache
         self.type = None
         self.version = None
@@ -36,66 +38,85 @@ def _initTypeAndVersion(self):
         self.type = compiler_type
         self.version = (major_ver, minor_ver, patchlevel)
 
-    def _basicCmd(self, infiles, out, is_link=False):
+    def _basicCmd(self, source_files, out, is_link=False):
         cmd = []
         if self.use_ccache and not is_link:
             cmd += ['ccache']
         cmd += [self.path]
         if out is not None:
             cmd += ['-o', out]
-        if isinstance(infiles, list):
-            cmd += infiles
-        elif isinstance(infiles, str):
-            cmd += [infiles]
+        if isinstance(source_files, list):
+            cmd += source_files
+        elif isinstance(source_files, str):
+            cmd += [source_files]
         else:
-            raise TypeError('infiles must be a string or list')
+            raise TypeError('source_files must be a string or list')
         return cmd
 
-    def preprocessCmd(self, infiles, out=None, flags=[]):
-        cmd = self._basicCmd(infiles, out) + ['-x', 'c++', '-E']
+    def preprocessCmd(self, source_files, out=None, flags=[]):
+        cmd = self._basicCmd(source_files, out) + ['-x', 'c++', '-E']
         cmd += self.flags + self.compile_flags + flags
         return cmd
 
-    def compileCmd(self, infiles, out=None, flags=[]):
-        cmd = self._basicCmd(infiles, out) + ['-x', 'c++', '-c']
+    def compileCmd(self, source_files, out=None, flags=[]):
+        cmd = self._basicCmd(source_files, out) + ['-x', 'c++', '-c']
         cmd += self.flags + self.compile_flags + flags
         return cmd
 
-    def linkCmd(self, infiles, out=None, flags=[]):
-        cmd = self._basicCmd(infiles, out, is_link=True)
+    def linkCmd(self, source_files, out=None, flags=[]):
+        cmd = self._basicCmd(source_files, out, is_link=True)
         cmd += self.flags + self.link_flags + flags
         return cmd
 
-    def compileLinkCmd(self, infiles, out=None, flags=[]):
-        cmd = self._basicCmd(infiles, out, is_link=True) + ['-x', 'c++']
+    def compileLinkCmd(self, source_files, out=None, flags=[]):
+        cmd = self._basicCmd(source_files, out, is_link=True) + ['-x', 'c++']
         cmd += self.flags + self.compile_flags + self.link_flags + flags
         return cmd
 
-    def preprocess(self, infiles, out=None, flags=[], env=None, cwd=None):
-        cmd = self.preprocessCmd(infiles, out, flags)
+    def preprocess(self, source_files, out=None, flags=[], env=None, cwd=None):
+        cmd = self.preprocessCmd(source_files, out, flags)
         out, err, rc = lit.util.executeCommand(cmd, env=env, cwd=cwd)
         return cmd, out, err, rc
 
-    def compile(self, infiles, out=None, flags=[], env=None, cwd=None):
-        cmd = self.compileCmd(infiles, out, flags)
+    def compile(self, source_files, out=None, flags=[], env=None, cwd=None):
+        cmd = self.compileCmd(source_files, out, flags)
         out, err, rc = lit.util.executeCommand(cmd, env=env, cwd=cwd)
         return cmd, out, err, rc
 
-    def link(self, infiles, out=None, flags=[], env=None, cwd=None):
-        cmd = self.linkCmd(infiles, out, flags)
+    def link(self, source_files, out=None, flags=[], env=None, cwd=None):
+        cmd = self.linkCmd(source_files, out, flags)
         out, err, rc = lit.util.executeCommand(cmd, env=env, cwd=cwd)
         return cmd, out, err, rc
 
-    def compileLink(self, infiles, out=None, flags=[], env=None, cwd=None):
-        cmd = self.compileLinkCmd(infiles, out, flags)
+    def compileLink(self, source_files, out=None, flags=[], env=None,
+                    cwd=None):
+        cmd = self.compileLinkCmd(source_files, out, flags)
         out, err, rc = lit.util.executeCommand(cmd, env=env, cwd=cwd)
         return cmd, out, err, rc
 
-    def dumpMacros(self, infiles=None, flags=[], env=None, cwd=None):
-        if infiles is None:
-            infiles = '/dev/null'
+    def compileLinkTwoSteps(self, source_file, out=None, object_file=None,
+                            flags=[], env=None, cwd=None):
+        if not isinstance(source_file, str):
+            raise TypeError('This function only accepts a single input file')
+        if object_file is None:
+            # Create, use and delete a temporary object file if none is given.
+            with_fn = lambda: libcxx.util.guardedTempFilename(suffix='.o')
+        else:
+            # Otherwise wrap the filename in a context manager function.
+            with_fn = lambda: libcxx.util.nullContext(object_file)
+        with with_fn() as object_file:
+            cmd, output, err, rc = self.compile(source_file, object_file,
+                                                flags=flags, env=env, cwd=cwd)
+            if rc != 0:
+                return cmd, output, err, rc
+            return self.link(object_file, out=out, flags=flags, env=env,
+                             cwd=cwd)
+
+    def dumpMacros(self, source_files=None, flags=[], env=None, cwd=None):
+        if source_files is None:
+            source_files = os.devnull
         flags = ['-dM'] + flags
-        cmd, out, err, rc = self.preprocess(infiles, flags=flags, env=env,
+        cmd, out, err, rc = self.preprocess(source_files, flags=flags, env=env,
                                             cwd=cwd)
         if rc != 0:
             return None

libcxx/test/libcxx/double_include.sh.cpp

@@ -0,0 +1,111 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// Test that we can include each header in two TU's and link them together.
+
+// RUN: %cxx -c %s -o %t.first.o %flags %compile_flags
+// RUN: %cxx -c %s -o %t.second.o -DWITH_MAIN %flags %compile_flags
+// RUN: %cxx -o %t.exe %t.first.o %t.second.o %flags %link_flags
+// RUN: %run
+
+#include <algorithm>
+#include <array>
+#include <bitset>
+#include <cassert>
+#include <ccomplex>
+#include <cctype>
+#include <cerrno>
+#include <cfenv>
+#include <cfloat>
+#include <chrono>
+#include <cinttypes>
+#include <ciso646>
+#include <climits>
+#include <clocale>
+#include <cmath>
+#include <codecvt>
+#include <complex>
+#include <complex.h>
+#include <condition_variable>
+#include <csetjmp>
+#include <csignal>
+#include <cstdarg>
+#include <cstdbool>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctgmath>
+#include <ctime>
+#include <cwchar>
+#include <cwctype>
+#include <deque>
+#include <exception>
+#include <experimental/dynarray>
+#include <experimental/optional>
+#include <experimental/string_view>
+#include <experimental/type_traits>
+#include <experimental/utility>
+#include <ext/hash_map>
+#include <ext/hash_set>
+#include <forward_list>
+#include <fstream>
+#include <functional>
+#include <initializer_list>
+#include <iomanip>
+#include <ios>
+#include <iosfwd>
+#include <iostream>
+#include <istream>
+#include <iterator>
+#include <limits>
+#include <list>
+#include <locale>
+#include <map>
+#include <memory>
+#include <new>
+#include <numeric>
+#include <ostream>
+#include <queue>
+#include <random>
+#include <ratio>
+#include <regex>
+#include <scoped_allocator>
+#include <set>
+#include <sstream>
+#include <stack>
+#include <stdexcept>
+#include <streambuf>
+#include <string>
+#include <strstream>
+#include <system_error>
+#include <tgmath.h>
+#include <tuple>
+#include <typeindex>
+#include <typeinfo>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <valarray>
+#include <vector>
+
+#ifndef _LIBCPP_HAS_NO_THREADS
+#include <atomic>
+#include <future>
+#include <mutex>
+#include <shared_mutex>
+#include <thread>
+#endif
+
+#if defined(WITH_MAIN)
+int main() {}
+#endif

libcxx/test/libcxx/selftest/not_test.sh.cpp

@@ -0,0 +1,17 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// RUN: %build
+// RUN: not %run
+
+int main()
+{
+  return 1;
+}

libcxx/test/libcxx/selftest/test.fail.cpp

@@ -0,0 +1,11 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#error This test should not compile.

libcxx/test/libcxx/selftest/test.pass.cpp

@@ -0,0 +1,13 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+int main()
+{
+}

libcxx/test/libcxx/selftest/test.sh.cpp

@@ -0,0 +1,16 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// RUN: %build
+// RUN: %run
+
+int main()
+{
+}