YAML: Enable the YAMLParser tests.

This commit enables the tests located in test/YAMLParser directory.
Those tests were never actually enabled, as llvm-lit didn't pick up the
files with the 'data' extension. The commit renames those test files to files
with the 'test' extension so that llvm-lit would find them.

This commit also modifies yaml-bench so that it returns an error status
if an error occurred during parsing. It also adds the '-use-color'
command line option to yaml-bench (to make sure that file check matches
the error messages in the output stream).

This commit modifies some of the renamed tests so that they wouldn't
fail. It gets rid of XFAILs and uses the 'not' command instead for
some of the tests that have to fail during parsing. This commit
also adds some 'FIXME' comments to a couple of tests that are
supposed to fail but currently pass because of various bugs
in the implementation of the yaml parser.

Reviewers: Justin Bogner

Differential Revision: http://reviews.llvm.org/D9448

llvm-svn: 236754

llvm/include/llvm/Support/YAMLParser.h

@@ -76,9 +76,9 @@ std::string escape(StringRef Input);
 class Stream {
 public:
   /// \brief This keeps a reference to the string referenced by \p Input.
-  Stream(StringRef Input, SourceMgr &);
+  Stream(StringRef Input, SourceMgr &, bool ShowColors = true);
 
-  Stream(MemoryBufferRef InputBuffer, SourceMgr &);
+  Stream(MemoryBufferRef InputBuffer, SourceMgr &, bool ShowColors = true);
   ~Stream();
 
   document_iterator begin();

llvm/lib/Support/YAMLParser.cpp

@@ -260,8 +260,8 @@ namespace yaml {
 /// @brief Scans YAML tokens from a MemoryBuffer.
 class Scanner {
 public:
-  Scanner(StringRef Input, SourceMgr &SM);
-  Scanner(MemoryBufferRef Buffer, SourceMgr &SM_);
+  Scanner(StringRef Input, SourceMgr &SM, bool ShowColors = true);
+  Scanner(MemoryBufferRef Buffer, SourceMgr &SM_, bool ShowColors = true);
 
   /// @brief Parse the next token and return it without popping it.
   Token &peekNext();
@@ -271,7 +271,7 @@ class Scanner {
 
   void printError(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Message,
                   ArrayRef<SMRange> Ranges = None) {
-    SM.PrintMessage(Loc, Kind, Message, Ranges);
+    SM.PrintMessage(Loc, Kind, Message, Ranges, /* FixIts= */ None, ShowColors);
   }
 
   void setError(const Twine &Message, StringRef::iterator Position) {
@@ -505,6 +505,9 @@ class Scanner {
   /// @brief True if an error has occurred.
   bool Failed;
 
+  /// @brief Should colors be used when printing out the diagnostic messages?
+  bool ShowColors;
+
   /// @brief Queue of tokens. This is required to queue up tokens while looking
   ///        for the end of a simple key. And for cases where a single character
   ///        can produce multiple tokens (e.g. BlockEnd).
@@ -706,11 +709,13 @@ std::string yaml::escape(StringRef Input) {
   return EscapedInput;
 }
 
-Scanner::Scanner(StringRef Input, SourceMgr &sm) : SM(sm) {
+Scanner::Scanner(StringRef Input, SourceMgr &sm, bool ShowColors)
+    : SM(sm), ShowColors(ShowColors) {
   init(MemoryBufferRef(Input, "YAML"));
 }
 
-Scanner::Scanner(MemoryBufferRef Buffer, SourceMgr &SM_) : SM(SM_) {
+Scanner::Scanner(MemoryBufferRef Buffer, SourceMgr &SM_, bool ShowColors)
+    : SM(SM_), ShowColors(ShowColors) {
   init(Buffer);
 }
 
@@ -1525,11 +1530,11 @@ bool Scanner::fetchMoreTokens() {
   return false;
 }
 
-Stream::Stream(StringRef Input, SourceMgr &SM)
-    : scanner(new Scanner(Input, SM)), CurrentDoc() {}
+Stream::Stream(StringRef Input, SourceMgr &SM, bool ShowColors)
+    : scanner(new Scanner(Input, SM, ShowColors)), CurrentDoc() {}
 
-Stream::Stream(MemoryBufferRef InputBuffer, SourceMgr &SM)
-    : scanner(new Scanner(InputBuffer, SM)), CurrentDoc() {}
+Stream::Stream(MemoryBufferRef InputBuffer, SourceMgr &SM, bool ShowColors)
+    : scanner(new Scanner(InputBuffer, SM, ShowColors)), CurrentDoc() {}
 
 Stream::~Stream() {}
 

llvm/test/YAMLParser/spec-05-02-utf8.data → llvm/test/YAMLParser/spec-05-02-utf8.test

@@ -1,4 +1,4 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
 
 # Invalid use of BOM
 # inside a

llvm/test/YAMLParser/spec-05-10.data → llvm/test/YAMLParser/spec-05-10.test

@@ -1,4 +1,4 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
 
 commercial-at: @text
 grave-accent: `text

llvm/test/YAMLParser/spec-05-12.data → llvm/test/YAMLParser/spec-05-12.test

@@ -1,7 +1,8 @@
 # RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
 #
+# FIXME: This test should actually fail. Yaml-bench should report an error
+# that a tab is being used to indent a plain scalar at line 15.
 # We don't currently reject tabs as indentation.
-# XFAIL: *
 
 # Tabs do's and don'ts:
 # comment:
@@ -13,4 +14,5 @@ block: |
 elsewhere:	# separation
 	indentation, in	plain scalar
 
-# CHECK: error
+
+# CHECK: !!str "Quoted\t\t"

llvm/test/YAMLParser/spec-05-15.test

@@ -0,0 +1,7 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+Bad escapes:
+  "\c
+  \xq-"
+
+# CHECK: error

llvm/test/YAMLParser/spec-07-03.test

@@ -0,0 +1,7 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+%YAML 1.1
+%YAML 1.1
+foo
+
+# CHECK: error

llvm/test/YAMLParser/spec-07-05.test

@@ -0,0 +1,8 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# We don't currently parse TAG directives.
+# CHECK: error: Unexpected token
+
+%TAG ! !foo
+%TAG ! !foo
+bar

llvm/test/YAMLParser/spec-08-04.test

@@ -0,0 +1,7 @@
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# We don't currently look at the content of literal tags.
+# CHECK: error: Unknown tag handle
+
+- !<!> foo
+- !<$:?> bar

llvm/test/YAMLParser/spec-08-06.data → llvm/test/YAMLParser/spec-08-06.test

@@ -1,12 +1,11 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
 #
 # We don't currently validate tags.
-# XFAIL: *
+# CHECK: error: Unknown tag handle
 
 %TAG !o! tag:ben-kiki.org,2000:
 ---
 - !$a!b foo
 - !o! bar
 - !h!type baz
 
-# CHECK: error

llvm/test/YAMLParser/spec-09-02.test

@@ -0,0 +1,14 @@
+# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+
+ "as space
+ trimmed
+
+ specific
+
+ escaped	\
+ none"
+
+# FIXME: The string below should actually be
+#   "as space trimmed\nspecific\nescaped\tnone", but the parser currently has
+#   a bug when parsing multiline quoted strings.
+# CHECK: !!str "as space\n trimmed\n specific\n escaped\t none"

llvm/test/YAMLParser/spec-09-14.test

@@ -0,0 +1,21 @@
+# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+#
+# FIXME: This test should actually fail. Yaml bench should report an error that
+# says that the '---' and '...' document start/end markers must not be specified
+# as the first content line of a non-indented plain scalar.
+# CHECK: !!str
+
+---
+--- ||| : foo
+... >>>: bar
+---
+[
+---
+,
+... ,
+{
+--- :
+... # Nested
+}
+]
+...

llvm/test/YAMLParser/spec-09-21.data → llvm/test/YAMLParser/spec-09-21.test

@@ -1,4 +1,4 @@
-# RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
+# RUN: not yaml-bench -canonical %s 2>&1 | FileCheck %s
 
 - |
 

llvm/test/YAMLParser/spec-10-08.data → llvm/test/YAMLParser/spec-10-08.test

@@ -1,13 +1,13 @@
 # RUN: yaml-bench -canonical %s 2>&1 | FileCheck %s
 #
-# This fails because even without a key token, some contexts (in this case flow
-# maps) allow implicit null keys, which mix with this in weird ways.
-# XFAIL: *
+# FIXME: This test should fail. Yaml bench should report an error that a simple
+# key spans across multiple lines and that another simple key is longer than
+# 1024 characters.
 
 {
 multi-line
  simple key : value,
 very long ...................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................(>1KB)................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................... key: value
 }
 
-# CHECK: error
+# CHECK: ? !!str "very long

llvm/utils/yaml-bench/YAMLBench.cpp

@@ -19,6 +19,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/Timer.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/raw_ostream.h"
 #include <system_error>
@@ -52,6 +53,10 @@ static cl::opt<unsigned>
                   "Do not use more megabytes of memory"),
                 cl::init(1000));
 
+cl::opt<cl::boolOrDefault>
+    UseColor("use-color", cl::desc("Emit colored output (default=autodetect)"),
+             cl::init(cl::BOU_UNSET));
+
 struct indent {
   unsigned distance;
   indent(unsigned d) : distance(d) {}
@@ -187,6 +192,9 @@ static std::string createJSONText(size_t MemoryMB, unsigned ValueSize) {
 
 int main(int argc, char **argv) {
   llvm::cl::ParseCommandLineOptions(argc, argv);
+  bool ShowColors = UseColor == cl::BOU_UNSET
+                        ? sys::Process::StandardOutHasColors()
+                        : UseColor == cl::BOU_TRUE;
   if (Input.getNumOccurrences()) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
         MemoryBuffer::getFileOrSTDIN(Input);
@@ -200,8 +208,10 @@ int main(int argc, char **argv) {
     }
 
     if (DumpCanonical) {
-      yaml::Stream stream(Buf.getBuffer(), sm);
+      yaml::Stream stream(Buf.getBuffer(), sm, ShowColors);
       dumpStream(stream);
+      if (stream.failed())
+        return 1;
     }
   }