Update testdata

In the past 6 months, `cucumber/cucumber`-repo has seen a variet of
changes all of which have been fixed with this commit:
  * background to have `id`
  * rule to have `id`
  * examples to have `id`
  * ensure to only strip leading and trailing spaces in table cells.
  Keep newline intact
  * ensure escaped delimiter within doc_string is not recorded as escaped
  * doc_string has field change: contentType -> mediaType
  * ignore comments on tags

See:
  cucumber/common#891
  cucumber/common#880
  cucumber/common#889
  cucumber/common#800

At the time of this commit, master for `cucumber/cucumber` was at:
a7c593f479e7ae739b49108f79bba4853352d99c

lib/ast_ndjson/ast_ndjson.ex

@@ -254,12 +254,12 @@ defmodule ExGherkin.AstNdjson do
     def doc_string_constituents_ctx(acc = Acc.storage()),
       do: Acc.set_local(acc, acc |> Acc.peek_ctx() |> elem(1))
 
-    def doc_string_constituents_reconcile({ctx, content_type, content}, acc = Acc.storage()) do
+    def doc_string_constituents_reconcile({ctx, media_type, content}, acc = Acc.storage()) do
       {raw_location, delimiter} = ctx
 
       local_acc =
         content
-        |> DocString.new(content_type, delimiter, Location.new(raw_location))
+        |> DocString.new(media_type, delimiter, Location.new(raw_location))
 
       Acc.set_local(acc, local_acc)
     end

lib/ast_ndjson/lib/background.ex

@@ -22,9 +22,10 @@ defmodule ExGherkin.AstNdjson.Background do
             location: Location.new(),
             steps: [],
             token: nil,
+            id: "",
             parsed_sentence: %{}
 
-  def new(name, description, keyword, location = %Location{}, steps, token) do
+  def new(name, description, keyword, location = %Location{}, steps, token, id \\ "0") do
     %{
       background:
         struct(__MODULE__, %{
@@ -34,6 +35,7 @@ defmodule ExGherkin.AstNdjson.Background do
           location: location,
           steps: Util.normalize(steps),
           token: token,
+          id: id,
           parsed_sentence: Util.parse_sentence(name),
         })
     }

lib/ast_ndjson/lib/doc_string.ex

@@ -16,14 +16,14 @@ defmodule ExGherkin.AstNdjson.DocString do
 
   defstruct location: Location.new(),
             content: "",
-            contentType: "",
+            mediaType: "",
             delimiter: ""
 
-  def new(content, content_type, delimiter, location = %Location{}) do
+  def new(content, media_type, delimiter, location = %Location{}) do
     struct(__MODULE__, %{
       location: location,
       content: Util.normalize(content),
-      contentType: Util.normalize(content_type),
+      mediaType: Util.normalize(media_type),
       delimiter: Util.normalize(delimiter)
     })
   end

lib/ast_ndjson/lib/examples.ex

@@ -27,9 +27,10 @@ defmodule ExGherkin.AstNdjson.Examples do
             tableHeader: %{},
             tableBody: %{},
             tags: [],
+            id: "",
             parsed_sentence: %{}
 
-  def new(name, description, keyword, tags, location = %Location{}, data_table_rows) do
+  def new(name, description, keyword, tags, location = %Location{}, data_table_rows, id \\ "0") do
     {header_section, body_section} = split_data_table_rows(data_table_rows)
 
     struct(__MODULE__, %{
@@ -40,6 +41,7 @@ defmodule ExGherkin.AstNdjson.Examples do
       tags: Util.normalize(tags),
       tableHeader: Util.normalize(header_section),
       tableBody: Util.normalize(body_section),
+      id: id,
       parsed_sentence: Util.parse_sentence(name),
     })
   end

lib/ast_ndjson/lib/rule.ex

@@ -21,9 +21,10 @@ defmodule ExGherkin.AstNdjson.Rule do
             keyword: "",
             children: [],
             token: nil,
+            id: "",
             parsed_sentence: %{}
 
-  def new(name, description, keyword, location = %Location{}, token) do
+  def new(name, description, keyword, location = %Location{}, token, id \\ "0") do
     %{
       rule:
         struct(__MODULE__, %{
@@ -32,6 +33,7 @@ defmodule ExGherkin.AstNdjson.Rule do
           keyword: Util.normalize(keyword),
           location: location,
           token: token,
+          id: id,
           parsed_sentence: Util.parse_sentence(name),
         })
     }

lib/scanner/scanner.ex

@@ -51,6 +51,7 @@ defmodule ExGherkin.Scanner do
     trimmed_line
     |> case do
       "\\\"\\\"\\\"" -> {handle_plain_text("\"\"\"", index, 1), context}
+      "\\`\\`\\`" -> {handle_plain_text("```", index, 1), context}
       "" -> {Token.content(index, 1, ""), context}
       _ -> {handle_plain_text(trimmed_line, index, 1), context}
     end
@@ -297,7 +298,7 @@ defmodule ExGherkin.Scanner do
       |> String.trim_trailing("|")
       |> Utils.data_table_pipe_splitter(column)
       |> Enum.map(fn {offset_count, e} ->
-        {offset_count, String.trim(e)}
+        {offset_count, String.trim(e, " ")}
       end)
 
     {Token.data_table(
@@ -311,6 +312,8 @@ defmodule ExGherkin.Scanner do
   def map_to_token(_, <<"@", rest::binary>>, index, column, context = %Context{}) do
     {_, text} =
       rest
+      |> String.split(" #")
+      |> List.first
       |> String.split("@")
       |> Enum.reduce({column, []}, fn tag, {left_offset, tags} ->
         {left_offset, trimmed_leading} = Utils.count_spaces_before(tag, left_offset)

test/support/testdata/bad/inconsistent_cell_count.feature.errors.ndjson

@@ -1,24 +1,2 @@
-{
-  "attachment": {
-    "data": "(6:7): inconsistent cell count within the table",
-    "source": {
-      "location": {
-        "column": 7,
-        "line": 6
-      },
-      "uri": "testdata/bad/inconsistent_cell_count.feature"
-    }
-  }
-}
-{
-  "attachment": {
-    "data": "(14:5): inconsistent cell count within the table",
-    "source": {
-      "location": {
-        "column": 5,
-        "line": 14
-      },
-      "uri": "testdata/bad/inconsistent_cell_count.feature"
-    }
-  }
-}
+{"parseError":{"message":"(6:7): inconsistent cell count within the table","source":{"location":{"column":7,"line":6},"uri":"testdata/bad/inconsistent_cell_count.feature"}}}
+{"parseError":{"message":"(14:5): inconsistent cell count within the table","source":{"location":{"column":5,"line":14},"uri":"testdata/bad/inconsistent_cell_count.feature"}}}

test/support/testdata/bad/invalid_language.feature.errors.ndjson

@@ -1,12 +1 @@
-{
-  "attachment": {
-    "data": "(1:1): Language not supported: no-such",
-    "source": {
-      "location": {
-        "column": 1,
-        "line": 1
-      },
-      "uri": "testdata/bad/invalid_language.feature"
-    }
-  }
-}
+{"parseError":{"message":"(1:1): Language not supported: no-such","source":{"location":{"column":1,"line":1},"uri":"testdata/bad/invalid_language.feature"}}}

test/support/testdata/bad/multiple_parser_errors.feature.errors.ndjson

@@ -1,24 +1,2 @@
-{
-  "attachment": {
-    "data": "(2:1): expected: #EOF, #Language, #TagLine, #FeatureLine, #Comment, #Empty, got 'invalid line here'",
-    "source": {
-      "location": {
-        "column": 1,
-        "line": 2
-      },
-      "uri": "testdata/bad/multiple_parser_errors.feature"
-    }
-  }
-}
-{
-  "attachment": {
-    "data": "(9:1): expected: #EOF, #TableRow, #DocStringSeparator, #StepLine, #TagLine, #ExamplesLine, #ScenarioLine, #RuleLine, #Comment, #Empty, got 'another invalid line here'",
-    "source": {
-      "location": {
-        "column": 1,
-        "line": 9
-      },
-      "uri": "testdata/bad/multiple_parser_errors.feature"
-    }
-  }
-}
+{"parseError":{"message":"(2:1): expected: #EOF, #Language, #TagLine, #FeatureLine, #Comment, #Empty, got 'invalid line here'","source":{"location":{"column":1,"line":2},"uri":"testdata/bad/multiple_parser_errors.feature"}}}
+{"parseError":{"message":"(9:1): expected: #EOF, #TableRow, #DocStringSeparator, #StepLine, #TagLine, #ExamplesLine, #ScenarioLine, #RuleLine, #Comment, #Empty, got 'another invalid line here'","source":{"location":{"column":1,"line":9},"uri":"testdata/bad/multiple_parser_errors.feature"}}}

test/support/testdata/bad/not_gherkin.feature.errors.ndjson

@@ -1,12 +1 @@
-{
-  "attachment": {
-    "data": "(1:1): expected: #EOF, #Language, #TagLine, #FeatureLine, #Comment, #Empty, got 'not gherkin'",
-    "source": {
-      "location": {
-        "column": 1,
-        "line": 1
-      },
-      "uri": "testdata/bad/not_gherkin.feature"
-    }
-  }
-}
+{"parseError":{"message":"(1:1): expected: #EOF, #Language, #TagLine, #FeatureLine, #Comment, #Empty, got 'not gherkin'","source":{"location":{"column":1,"line":1},"uri":"testdata/bad/not_gherkin.feature"}}}

test/support/testdata/bad/single_parser_error.feature.errors.ndjson

@@ -1,12 +1 @@
-{
-  "attachment": {
-    "data": "(2:1): expected: #EOF, #Language, #TagLine, #FeatureLine, #Comment, #Empty, got 'invalid line here'",
-    "source": {
-      "location": {
-        "column": 1,
-        "line": 2
-      },
-      "uri": "testdata/bad/single_parser_error.feature"
-    }
-  }
-}
+{"parseError":{"message":"(2:1): expected: #EOF, #Language, #TagLine, #FeatureLine, #Comment, #Empty, got 'invalid line here'","source":{"location":{"column":1,"line":2},"uri":"testdata/bad/single_parser_error.feature"}}}

test/support/testdata/bad/unexpected_eof.feature.errors.ndjson

@@ -1,11 +1 @@
-{
-  "attachment": {
-    "data": "(7:0): unexpected end of file, expected: #TagLine, #ScenarioLine, #Comment, #Empty",
-    "source": {
-      "location": {
-        "line": 7
-      },
-      "uri": "testdata/bad/unexpected_eof.feature"
-    }
-  }
-}
+{"parseError":{"message":"(7:0): unexpected end of file, expected: #TagLine, #ScenarioLine, #Comment, #Empty","source":{"location":{"line":7},"uri":"testdata/bad/unexpected_eof.feature"}}}

test/support/testdata/bad/whitespace_in_tags.feature

@@ -0,0 +1,4 @@
+Feature: Whitespace in tags
+
+  @a tag containing whitespace
+  Scenario: minimalistic

test/support/testdata/bad/whitespace_in_tags.feature.errors.ndjson

@@ -0,0 +1 @@
+{"parseError":{"message":"(3:3): A tag may not contain whitespace","source":{"location":{"column":3,"line":3},"uri":"testdata/bad/whitespace_in_tags.feature"}}}

test/support/testdata/full_features/200-get-employee.feature.ast.ndjson

@@ -1 +1 @@
-{"gherkinDocument":{"uri":"/Users/kevinjohnson/projects/ex_gherkin/test/support/testdata/full_features/200-get-employee.feature","feature":{"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"Get Employee","children":[{"background":{"location":{"line":3,"column":3},"keyword":"Background","steps":[{"location":{"line":5,"column":5},"keyword":"Given ","text":"an employee with the following attributes","dataTable":{"location":{"line":6,"column":7},"rows":[{"location":{"line":6,"column":7},"cells":[{"location":{"line":6,"column":9},"value":"id"},{"location":{"line":6,"column":15},"value":"firstName"},{"location":{"line":6,"column":27},"value":"lastName"},{"location":{"line":6,"column":38},"value":"dateOfBirth"},{"location":{"line":6,"column":52},"value":"startDate"},{"location":{"line":6,"column":65},"value":"employmentType"},{"location":{"line":6,"column":82},"value":"email"}],"id":"0"},{"location":{"line":7,"column":7},"cells":[{"location":{"line":7,"column":9},"value":"200"},{"location":{"line":7,"column":15},"value":"Rachel"},{"location":{"line":7,"column":27},"value":"Green"},{"location":{"line":7,"column":38},"value":"1990-01-01"},{"location":{"line":7,"column":52},"value":"2018-01-01"},{"location":{"line":7,"column":65},"value":"Permanent"},{"location":{"line":7,"column":82},"value":"rachel.green@fs.com"}],"id":"1"}]},"id":"2"},{"location":{"line":9,"column":5},"keyword":"And ","text":"with the following phone numbers","dataTable":{"location":{"line":10,"column":7},"rows":[{"location":{"line":10,"column":7},"cells":[{"location":{"line":10,"column":9},"value":"id"},{"location":{"line":10,"column":15},"value":"type"},{"location":{"line":10,"column":24},"value":"isdCode"},{"location":{"line":10,"column":34},"value":"phoneNumber"},{"location":{"line":10,"column":48},"value":"extension"}],"id":"3"},{"location":{"line":11,"column":7},"cells":[{"location":{"line":11,"column":9},"value":"201"},{"location":{"line":11,"column":15},"value":"Mobile"},{"location":{"line":11,"column":24},"value":"+1"},{"location":{"line":11,"column":34},"value":"2141112222"},{"location":{"line":11,"column":58}}],"id":"4"},{"location":{"line":12,"column":7},"cells":[{"location":{"line":12,"column":9},"value":"202"},{"location":{"line":12,"column":15},"value":"Office"},{"location":{"line":12,"column":24},"value":"+1"},{"location":{"line":12,"column":34},"value":"8362223000"},{"location":{"line":12,"column":48},"value":"333"}],"id":"5"}]},"id":"6"},{"location":{"line":14,"column":5},"keyword":"When ","text":"employee already exists","id":"7"}]}},{"scenario":{"location":{"line":17,"column":3},"keyword":"Scenario","name":"GET BY ID","steps":[{"location":{"line":19,"column":5},"keyword":"When ","text":"user wants to get employee by id 200","id":"8"},{"location":{"line":20,"column":5},"keyword":"Then ","text":"the get 'IS SUCCESSFUL'","id":"9"},{"location":{"line":21,"column":5},"keyword":"And ","text":"following employee is returned","dataTable":{"location":{"line":22,"column":7},"rows":[{"location":{"line":22,"column":7},"cells":[{"location":{"line":22,"column":9},"value":"id"},{"location":{"line":22,"column":15},"value":"firstName"},{"location":{"line":22,"column":27},"value":"lastName"},{"location":{"line":22,"column":38},"value":"dateOfBirth"},{"location":{"line":22,"column":52},"value":"startDate"},{"location":{"line":22,"column":65},"value":"employmentType"},{"location":{"line":22,"column":82},"value":"email"}],"id":"10"},{"location":{"line":23,"column":7},"cells":[{"location":{"line":23,"column":9},"value":"200"},{"location":{"line":23,"column":15},"value":"Rachel"},{"location":{"line":23,"column":27},"value":"Green"},{"location":{"line":23,"column":38},"value":"1990-01-01"},{"location":{"line":23,"column":52},"value":"2018-01-01"},{"location":{"line":23,"column":65},"value":"Permanent"},{"location":{"line":23,"column":82},"value":"rachel.green@fs.com"}],"id":"11"}]},"id":"12"},{"location":{"line":25,"column":5},"keyword":"And ","text":"following employee phone numbers are returned","dataTable":{"location":{"line":26,"column":7},"rows":[{"location":{"line":26,"column":7},"cells":[{"location":{"line":26,"column":9},"value":"id"},{"location":{"line":26,"column":15},"value":"type"},{"location":{"line":26,"column":24},"value":"isdCode"},{"location":{"line":26,"column":34},"value":"phoneNumber"},{"location":{"line":26,"column":48},"value":"extension"}],"id":"13"},{"location":{"line":27,"column":7},"cells":[{"location":{"line":27,"column":9},"value":"201"},{"location":{"line":27,"column":15},"value":"Mobile"},{"location":{"line":27,"column":24},"value":"+1"},{"location":{"line":27,"column":34},"value":"2141112222"},{"location":{"line":27,"column":58}}],"id":"14"},{"location":{"line":28,"column":7},"cells":[{"location":{"line":28,"column":9},"value":"202"},{"location":{"line":28,"column":15},"value":"Office"},{"location":{"line":28,"column":24},"value":"+1"},{"location":{"line":28,"column":34},"value":"8362223000"},{"location":{"line":28,"column":48},"value":"333"}],"id":"15"}]},"id":"16"}],"id":"17"}}]}}}
+{"gherkinDocument":{"feature":{"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"Get Employee","children":[{"background":{"id":"8","location":{"line":3,"column":3},"keyword":"Background","steps":[{"location":{"line":5,"column":5},"keyword":"Given ","text":"an employee with the following attributes","dataTable":{"location":{"line":6,"column":7},"rows":[{"id":"0","location":{"line":6,"column":7},"cells":[{"location":{"line":6,"column":9},"value":"id"},{"location":{"line":6,"column":15},"value":"firstName"},{"location":{"line":6,"column":27},"value":"lastName"},{"location":{"line":6,"column":38},"value":"dateOfBirth"},{"location":{"line":6,"column":52},"value":"startDate"},{"location":{"line":6,"column":65},"value":"employmentType"},{"location":{"line":6,"column":82},"value":"email"}]},{"id":"1","location":{"line":7,"column":7},"cells":[{"location":{"line":7,"column":9},"value":"200"},{"location":{"line":7,"column":15},"value":"Rachel"},{"location":{"line":7,"column":27},"value":"Green"},{"location":{"line":7,"column":38},"value":"1990-01-01"},{"location":{"line":7,"column":52},"value":"2018-01-01"},{"location":{"line":7,"column":65},"value":"Permanent"},{"location":{"line":7,"column":82},"value":"rachel.green@fs.com"}]}]},"id":"2"},{"location":{"line":9,"column":5},"keyword":"And ","text":"with the following phone numbers","dataTable":{"location":{"line":10,"column":7},"rows":[{"id":"3","location":{"line":10,"column":7},"cells":[{"location":{"line":10,"column":9},"value":"id"},{"location":{"line":10,"column":15},"value":"type"},{"location":{"line":10,"column":24},"value":"isdCode"},{"location":{"line":10,"column":34},"value":"phoneNumber"},{"location":{"line":10,"column":48},"value":"extension"}]},{"id":"4","location":{"line":11,"column":7},"cells":[{"location":{"line":11,"column":9},"value":"201"},{"location":{"line":11,"column":15},"value":"Mobile"},{"location":{"line":11,"column":24},"value":"+1"},{"location":{"line":11,"column":34},"value":"2141112222"},{"location":{"line":11,"column":58}}]},{"id":"5","location":{"line":12,"column":7},"cells":[{"location":{"line":12,"column":9},"value":"202"},{"location":{"line":12,"column":15},"value":"Office"},{"location":{"line":12,"column":24},"value":"+1"},{"location":{"line":12,"column":34},"value":"8362223000"},{"location":{"line":12,"column":48},"value":"333"}]}]},"id":"6"},{"location":{"line":14,"column":5},"keyword":"When ","text":"employee already exists","id":"7"}]}},{"scenario":{"id":"18","location":{"line":17,"column":3},"keyword":"Scenario","name":"GET BY ID","steps":[{"location":{"line":19,"column":5},"keyword":"When ","text":"user wants to get employee by id 200","id":"9"},{"location":{"line":20,"column":5},"keyword":"Then ","text":"the get 'IS SUCCESSFUL'","id":"10"},{"location":{"line":21,"column":5},"keyword":"And ","text":"following employee is returned","dataTable":{"location":{"line":22,"column":7},"rows":[{"id":"11","location":{"line":22,"column":7},"cells":[{"location":{"line":22,"column":9},"value":"id"},{"location":{"line":22,"column":15},"value":"firstName"},{"location":{"line":22,"column":27},"value":"lastName"},{"location":{"line":22,"column":38},"value":"dateOfBirth"},{"location":{"line":22,"column":52},"value":"startDate"},{"location":{"line":22,"column":65},"value":"employmentType"},{"location":{"line":22,"column":82},"value":"email"}]},{"id":"12","location":{"line":23,"column":7},"cells":[{"location":{"line":23,"column":9},"value":"200"},{"location":{"line":23,"column":15},"value":"Rachel"},{"location":{"line":23,"column":27},"value":"Green"},{"location":{"line":23,"column":38},"value":"1990-01-01"},{"location":{"line":23,"column":52},"value":"2018-01-01"},{"location":{"line":23,"column":65},"value":"Permanent"},{"location":{"line":23,"column":82},"value":"rachel.green@fs.com"}]}]},"id":"13"},{"location":{"line":25,"column":5},"keyword":"And ","text":"following employee phone numbers are returned","dataTable":{"location":{"line":26,"column":7},"rows":[{"id":"14","location":{"line":26,"column":7},"cells":[{"location":{"line":26,"column":9},"value":"id"},{"location":{"line":26,"column":15},"value":"type"},{"location":{"line":26,"column":24},"value":"isdCode"},{"location":{"line":26,"column":34},"value":"phoneNumber"},{"location":{"line":26,"column":48},"value":"extension"}]},{"id":"15","location":{"line":27,"column":7},"cells":[{"location":{"line":27,"column":9},"value":"201"},{"location":{"line":27,"column":15},"value":"Mobile"},{"location":{"line":27,"column":24},"value":"+1"},{"location":{"line":27,"column":34},"value":"2141112222"},{"location":{"line":27,"column":58}}]},{"id":"16","location":{"line":28,"column":7},"cells":[{"location":{"line":28,"column":9},"value":"202"},{"location":{"line":28,"column":15},"value":"Office"},{"location":{"line":28,"column":24},"value":"+1"},{"location":{"line":28,"column":34},"value":"8362223000"},{"location":{"line":28,"column":48},"value":"333"}]}]},"id":"17"}]}}]},"uri":"/Users/kevinjohnson/projects/ex_gherkin/test/support/testdata/full_features/200-get-employee.feature"}}