[wip] event-protocol: Add events & schemas for test runs #172

Open
wants to merge 9 commits into
from
View
@@ -1,25 +1,13 @@
-EXAMPLES = $(wildcard examples/events/*.json)
-SCHEMAS = $(wildcard schemas/*.json)
-SRC = $(wildcard bin/* lib/*)
+MAKEFILES=validator/Makefile ruby/Makefile
-default: .valid
- yarn link
+default: $(patsubst %/Makefile,default-%,$(MAKEFILES))
.PHONY: default
-.built: yarn.lock $(SRC)
- yarn test
- touch $@
+default-%: %
+ cd $< && make default
-yarn.lock: package.json
- yarn install
-
-clean:
- rm -rf yarn.lock node_modules coverage dist .built .valid examples/events.ndjson
+clean: $(patsubst %/Makefile,clean-%,$(MAKEFILES))
.PHONY: clean
-.valid: examples/events.ndjson yarn.lock ./bin/cucumber-event-validator.js $(SCHEMAS) $(SRC)
- @cat $< | node ./bin/cucumber-event-validator.js
- @touch $@
-
-examples/events.ndjson: $(EXAMPLES)
- @cat $(EXAMPLES) | jq --compact-output "." > $@
+clean-%: %
+ cd $< && make clean
@@ -1,5 +0,0 @@
-{"data":"Feature: Hello\n Scenario: World\n Given a step\n\n","media":{"encoding":"utf-8","type":"text/vnd.cucumber.gherkin+plain"},"type":"source","uri":"features/hello.feature"}
-{"document":{"comments":[],"feature":{"children":[{"keyword":"Scenario","location":{"column":3,"line":2},"name":"World","steps":[{"keyword":"Given ","location":{"column":5,"line":3},"text":"a step","type":"Step"}],"tags":[],"type":"Scenario"}],"keyword":"Feature","language":"en","location":{"column":1,"line":1},"name":"Hello","tags":[],"type":"Feature"},"type":"GherkinDocument"},"type":"gherkin-document","uri":"features/hello.feature"}
-{"pickle":{"language":"en","locations":[{"column":3,"line":2}],"name":"World","steps":[{"arguments":[],"locations":[{"column":11,"line":3}],"text":"a step"}],"tags":[]},"type":"pickle","uri":"features/hello.feature"}
-{"type":"attachment","source":{"uri":"features/hello.feature","start":{"line":3,"column":7}},"data":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAELUExURUdwTACoGAClFACqHACoFwCnGQCnFQCnGACoGQCoGQCoGACuGgCnFwCiFwCoGACoGACnFwClFgCpGACoFwD/AACnGAC2JACoGACnFwCqKgCoGACoGACqGQCoFwCnFwCoFwB/AACnGACoGAC/AACoGACoFgCmFwCnFwCnFwCqFgCmFgCZAACoGC64Ql3IbA2sJBuxMBiwLhewLR2yMgmrIOn36/z+/BGuJ9/04rHkuIDUjGHJcA6tJeD04wGoGQusIpnco+T257PlugSpHN704XDOfQeqHiCzNc7u0+z57t3z4HvSh7fmvj69UJzdpSi2PNjy3AKpGlrHahqxMBCtJpHZm83u0jS6R0vCXEWczDEAAAAsdFJOUwDIJQlYPSP7cFCqE1cW+mf4ImiBAXQHPu8GfP0zzoyOAp2wBOg4a2PqOU4FxgPjdgAAAK9JREFUGNNNj1UCwkAMRBcotLi7uw1S3N3d4f4nYSvIfO28bCYJIaI0OSabJ1+pFaDKpD4+zUJS1CHVWSwer+esDNhEQP+fV/f5uAd4XEIesNldl5WK0KSlQAtcDkc5xUcBA+y3tL/UKQIhCpLA+sTzw1K7UQVHQTyB23QyKraaNSAgTOGAQb9bqBeAoDg2EpMT4XdLm3rD8qrO7zFWO7UW8+86oodJp/zzRGUwyq83Mjcb8VXl0ZMAAAAASUVORK5CYII=","media":{"encoding":"base64","type":"image/png"}}
-{"type":"attachment","source":{"uri":"features/hello.feature","start":{"line":3,"column":7}},"data":"Exception in thread \"main\" java.lang.NullPointerException\n at com.example.myproject.Book.getTitle(Book.java:16)\n at com.example.myproject.Author.getBookTitles(Author.java:25)\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)\n","media":{"encoding":"utf-8","type":"text/vnd.cucumber.stacktrace.java+plain"}}
@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/Gemfile.local
@@ -0,0 +1,2 @@
+--format documentation
+--color
@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+ - 2.1.5
+before_install: gem install bundler -v 1.13.1
@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'cucumber', git: 'https://github.com/cucumber/cucumber-ruby.git', branch: 'event-stream-3'
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) Cucumber Limited and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
@@ -0,0 +1,11 @@
+default: ../validator/yarn.lock Gemfile.lock test
+
+Gemfile.lock: Gemfile
+ bundle install
+
+test:
+ bundle exec rake
+.PHONY: test
+
+clean:
+ rm Gemfile.lock
@@ -0,0 +1,25 @@
+# Cucumber EventStream Formatter
+
+This is a plugin for Cucumber-Ruby that emits the Cucumber [event protocol](https://github.com/cucumber/cucumber/blob/master/event-protocol).
+
+## Installation
+
+This gem is not designed to be installed stand-alone. It is a component used within the `cucumber` gem.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake` to run the tests.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/cucumber/cucumber
+
+## Code of Conduct
+
+Everyone interacting in this codebase and issue tracker is expected to follow the Cucumber [code of conduct](https://github.com/cucumber/cucumber/blob/master/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here
@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cucumber/events/version'
+
+Gem::Specification.new do |spec|
+ spec.name = "cucumber-events"
+ spec.version = Cucumber::Events::VERSION
+ spec.authors = ["Matt Wynne"]
+ spec.email = ["matt@cucumber.io"]
+
+ spec.summary = %q{Library for events emitted by cucumber-ruby}
+ spec.description = %q{Streams events that describe your test run in real-time.}
+ spec.homepage = "https://cucumber.io"
+ spec.license = "MIT"
+
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
+ f.match(%r{^(test|spec|features)/})
+ end
+ spec.bindir = "exe"
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+ spec.require_paths = ["lib"]
+
+ spec.add_development_dependency "bundler", "~> 1.13"
+ spec.add_development_dependency "rake", "~> 10.0"
+ spec.add_development_dependency "rspec", "~> 3.0"
+ spec.add_development_dependency "cucumber", "~> 3.0.0.pre.2"
+ spec.add_development_dependency "json", "~> 1.8"
+ spec.add_development_dependency "io-console", "~> 0.4.2"
+end
@@ -0,0 +1,11 @@
+{"type":"source","uri":"features/failing.feature","data":"Feature: Failing feature\n Scenario: failling\n Given this step fails\n Then this step will be skipped\n","media":{"encoding":"utf-8","type":"text/vnd.cucumber.gherkin+plain"}}
+{"type":"test-run-started","workingDirectory":"{{Dir.pwd}}/examples/a-failing-scenario","timestamp":1490450321}
+{"type":"test-case-prepared","sourceLocation":{"uri":"features/failing.feature","line":2},"steps":[{"actionLocation":{"uri":"{{Cucumber::LIBDIR}}/cucumber/filters/prepare_world.rb","line":28}},{"actionLocation":{"uri":"step_definitions/steps.rb","line":1},"sourceLocation":{"uri":"features/failing.feature","line":3}},{"actionLocation":{"uri":"step_definitions/steps.rb","line":5},"sourceLocation":{"uri":"features/failing.feature","line":4}}]}
+{"type":"test-case-started","sourceLocation":{"uri":"features/failing.feature","line":2}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/failing.feature","line":2}},"index":0}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/failing.feature","line":2}},"index":0,"result":{"status":"passed","duration":3000}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/failing.feature","line":2}},"index":1}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/failing.feature","line":2}},"index":1,"result":{"status":"failed","duration":140000,"exception":{"message":"Failing step","type":"RuntimeError","stackTrace":["{{Dir.pwd}}/examples/a-failing-scenario/step_definitions/steps.rb:2:in `/^this step fails$/'","features/failing.feature:3:in `Given this step fails'"]}}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/failing.feature","line":2}},"index":2}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/failing.feature","line":2}},"index":2,"result":{"status":"skipped","exception":{"message":"","type":"Cucumber::Core::Test::Result::Skipped","stackTrace":[]}}}
+{"type":"test-case-finished","sourceLocation":{"uri":"features/failing.feature","line":2},"result":{"status":"failed","duration":12344000,"exception":{"message":"Failing step","type":"RuntimeError","stackTrace":["{{Dir.pwd}}/examples/a-failing-scenario/step_definitions/steps.rb:2:in `/^this step fails$/'","features/failing.feature:3:in `Given this step fails'"]}}}
@@ -0,0 +1,4 @@
+Feature: Failing feature
+ Scenario: failling
+ Given this step fails
+ Then this step will be skipped
@@ -0,0 +1,7 @@
+Given(/^this step fails$/) do
+ raise "Failing step"
+end
+
+Then(/^this step will be skipped$/) do
+ # noop
+end
@@ -0,0 +1,9 @@
+{"type":"source","uri":"features/passing.feature","data":"Feature: Passing feature\n Scenario: Passing scenario\n Given this step passes\n","media":{"encoding":"utf-8","type":"text/vnd.cucumber.gherkin+plain"}}
+{"type":"test-run-started","workingDirectory":"{{Dir.pwd}}/examples/a-single-passing-scenario","timestamp":1490444711}
+{"type":"test-case-prepared","sourceLocation":{"uri":"features/passing.feature","line":2},"steps":[{"actionLocation":{"uri":"{{Cucumber::LIBDIR}}/cucumber/filters/prepare_world.rb","line":28}},{"actionLocation":{"uri":"step_definitions/steps.rb","line":1},"sourceLocation":{"uri":"features/passing.feature","line":3}}]}
+{"type":"test-case-started","sourceLocation":{"uri":"features/passing.feature","line":2}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":0}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":0,"result":{"status":"passed","duration":3000}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":1}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/passing.feature","line":2}},"index":1,"result":{"status":"passed","duration":381000}}
+{"type":"test-case-finished","sourceLocation":{"uri":"features/passing.feature","line":2},"result":{"status":"passed","duration":24008000}}
@@ -0,0 +1,3 @@
+Feature: Passing feature
+ Scenario: Passing scenario
+ Given this step passes
@@ -0,0 +1,3 @@
+Given(/^this step passes$/) do
+ # noop
+end
@@ -0,0 +1,9 @@
+{"type":"source","uri":"features/undefined.feature","data":"Feature: Undefined feature\n Scenario: Undefined scenario\n Given this step is undefined\n","media":{"encoding":"utf-8","type":"text/vnd.cucumber.gherkin+plain"}}
+{"type":"test-run-started","workingDirectory":"{{Dir.pwd}}/examples/an-undefined-step","timestamp":1490444711}
+{"type":"test-case-prepared","sourceLocation":{"uri":"features/undefined.feature","line":2},"steps":[{"actionLocation":{"uri":"{{Cucumber::LIBDIR}}/cucumber/filters/prepare_world.rb","line":28}},{"sourceLocation":{"uri":"features/undefined.feature","line":3}}]}
+{"type":"test-case-started","sourceLocation":{"uri":"features/undefined.feature","line":2}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/undefined.feature","line":2}},"index":0}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/undefined.feature","line":2}},"index":0,"result":{"status":"passed","duration":3000}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/undefined.feature","line":2}},"index":1}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/undefined.feature","line":2}},"index":1,"result":{"status":"undefined","exception":{"message":"Undefined step: \"this step is undefined\"","type":"Cucumber::Core::Test::Result::Undefined","stackTrace":["features/undefined.feature:3:in `Given this step is undefined'"]}}}
+{"type":"test-case-finished","sourceLocation":{"uri":"features/undefined.feature","line":2},"result":{"status":"undefined","duration":12811000,"exception":{"message":"Undefined step: \"this step is undefined\"","type":"Cucumber::Core::Test::Result::Undefined","stackTrace":["features/undefined.feature:3:in `Given this step is undefined'"]}}}
@@ -0,0 +1,3 @@
+Feature: Undefined feature
+ Scenario: Undefined scenario
+ Given this step is undefined
@@ -0,0 +1,11 @@
+Each of these examples isolates a simple case of using Cucumber with the events plugin, showing
+the expected events output in the `expected-events.ndjson`.
+
+These are used in the acceptance tests in `spec/cucumber/formatter/event_stream/event_stream_spec.rb` and the real output from running cucumber is comapred with this expected output.
+To run a test manually, `cd` into the directory of the example, then run:
+
+```
+bundle exec cucumber -r . --format Cucumber::Formatter::EventStream::Plugin
+```
+
+Note that a certain amount of normalisation (for timestamps and directory paths) is neccesary before comparing the output for testing.
@@ -0,0 +1,9 @@
+{"type":"source","uri":"features/passing_outline.feature","data":"Feature:\n Scenario Outline:\n Given this step is <status>\n\n Examples:\n | status |\n | passing |\n","media":{"encoding":"utf-8","type":"text/vnd.cucumber.gherkin+plain"}}
+{"type":"test-run-started","workingDirectory":"{{Dir.pwd}}/examples/scenario-outline","timestamp":1494880725}
+{"type":"test-case-prepared","sourceLocation":{"uri":"features/passing_outline.feature","line":7},"steps":[{"actionLocation":{"uri":"{{Cucumber::LIBDIR}}/cucumber/filters/prepare_world.rb","line":28}},{"actionLocation":{"uri":"features/step_definitions/steps.rb","line":1},"sourceLocation":{"uri":"features/passing_outline.feature","line":7}}]}
+{"type":"test-case-started","sourceLocation":{"uri":"features/passing_outline.feature","line":7}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/passing_outline.feature","line":7}},"index":0}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/passing_outline.feature","line":7}},"index":0,"result":{"status":"passed","duration":5000}}
+{"type":"test-step-started","testCase":{"sourceLocation":{"uri":"features/passing_outline.feature","line":7}},"index":1}
+{"type":"test-step-finished","testCase":{"sourceLocation":{"uri":"features/passing_outline.feature","line":7}},"index":1,"result":{"status":"passed","duration":200000}}
+{"type":"test-case-finished","sourceLocation":{"uri":"features/passing_outline.feature","line":7},"result":{"status":"passed","duration":38645000}}
@@ -0,0 +1,7 @@
+Feature:
+ Scenario Outline:
+ Given this step is <status>
+
+ Examples:
+ | status |
+ | passing |
@@ -0,0 +1,3 @@
+Given(/pass/) do
+ # noop
+end
Oops, something went wrong.