Made test repeatability more consistent, added handled feature

bugsnag · Mar 8, 2018 · 589795a · 589795a
1 parent b88a561
commit 589795a
Show file tree

Hide file tree

Showing 22 changed files with 187 additions and 78 deletions.
diff --git a/.gitignore b/.gitignore
@@ -50,3 +50,6 @@ vendor
 # For rubinius:
 #*.rbc
 bin
+
+# For Bugsnag-Maze-Runner:
+maze_output
diff --git a/features/fixtures/plain/.dockerignore b/features/fixtures/plain/.dockerignore
@@ -1 +1,2 @@
 Gemfile.lock
+requests
diff --git a/features/fixtures/plain/Dockerfile b/features/fixtures/plain/Dockerfile
@@ -6,7 +6,7 @@ ARG APP_PATH
 RUN mkdir -p $APP_PATH
 WORKDIR $APP_PATH
 
-COPY Gemfile $APP_PATH
+COPY app/Gemfile $APP_PATH
 RUN bundle install
 
-COPY . $APP_PATH
+COPY app $APP_PATH
diff --git a/features/fixtures/plain/Gemfile → features/fixtures/plain/app/Gemfile b/features/fixtures/plain/Gemfile → features/fixtures/plain/app/Gemfile
diff --git a/features/fixtures/plain/app.rb → features/fixtures/plain/app/app.rb b/features/fixtures/plain/app.rb → features/fixtures/plain/app/app.rb
diff --git a/features/fixtures/plain/app/handled/IgnoreException.rb b/features/fixtures/plain/app/handled/IgnoreException.rb
@@ -0,0 +1,12 @@
+require './app'
+
+class SkippableError < RuntimeError
+  attr_accessor :skip_bugsnag
+end
+
+begin
+  raise SkippableError.new("NotifyException")
+rescue => exception
+  exception.skip_bugsnag = true
+  Bugsnag.notify(exception)
+end
diff --git a/features/fixtures/plain/app/handled/NotifyException.rb b/features/fixtures/plain/app/handled/NotifyException.rb
@@ -0,0 +1,9 @@
+require './app'
+
+configure_basics
+
+begin
+  raise RuntimeError.new("NotifyException")
+rescue => exception
+  Bugsnag.notify(exception)
+end
diff --git a/features/fixtures/plain/app/handled/NotifyString.rb b/features/fixtures/plain/app/handled/NotifyString.rb
@@ -0,0 +1,9 @@
+require './app'
+
+configure_basics
+
+begin
+  raise RuntimeError.new("NotifyException")
+rescue => exception
+  Bugsnag.notify("NotifyString")
+end
diff --git a/...res/fixtures/plain/unhandled/BadSyntax.rb → ...fixtures/plain/app/unhandled/BadSyntax.rb b/...res/fixtures/plain/unhandled/BadSyntax.rb → ...fixtures/plain/app/unhandled/BadSyntax.rb
diff --git a/...s/fixtures/plain/unhandled/CustomError.rb → ...xtures/plain/app/unhandled/CustomError.rb b/...s/fixtures/plain/unhandled/CustomError.rb → ...xtures/plain/app/unhandled/CustomError.rb
diff --git a/...res/fixtures/plain/unhandled/Interrupt.rb → ...fixtures/plain/app/unhandled/Interrupt.rb b/...res/fixtures/plain/unhandled/Interrupt.rb → ...fixtures/plain/app/unhandled/Interrupt.rb
diff --git a/...res/fixtures/plain/unhandled/LoadError.rb → ...fixtures/plain/app/unhandled/LoadError.rb b/...res/fixtures/plain/unhandled/LoadError.rb → ...fixtures/plain/app/unhandled/LoadError.rb
diff --git a/...ixtures/plain/unhandled/LocalJumpError.rb → ...res/plain/app/unhandled/LocalJumpError.rb b/...ixtures/plain/unhandled/LocalJumpError.rb → ...res/plain/app/unhandled/LocalJumpError.rb
diff --git a/...res/fixtures/plain/unhandled/NameError.rb → ...fixtures/plain/app/unhandled/NameError.rb b/...res/fixtures/plain/unhandled/NameError.rb → ...fixtures/plain/app/unhandled/NameError.rb
diff --git a/...fixtures/plain/unhandled/NoMethodError.rb → ...ures/plain/app/unhandled/NoMethodError.rb b/...fixtures/plain/unhandled/NoMethodError.rb → ...ures/plain/app/unhandled/NoMethodError.rb
diff --git a/.../fixtures/plain/unhandled/RuntimeError.rb → ...tures/plain/app/unhandled/RuntimeError.rb b/.../fixtures/plain/unhandled/RuntimeError.rb → ...tures/plain/app/unhandled/RuntimeError.rb
diff --git a/...s/fixtures/plain/unhandled/SyntaxError.rb → ...xtures/plain/app/unhandled/SyntaxError.rb b/...s/fixtures/plain/unhandled/SyntaxError.rb → ...xtures/plain/app/unhandled/SyntaxError.rb
diff --git a/...xtures/plain/unhandled/SystemCallError.rb → ...es/plain/app/unhandled/SystemCallError.rb b/...xtures/plain/unhandled/SystemCallError.rb → ...es/plain/app/unhandled/SystemCallError.rb
diff --git a/...es/fixtures/plain/unhandled/SystemExit.rb → ...ixtures/plain/app/unhandled/SystemExit.rb b/...es/fixtures/plain/unhandled/SystemExit.rb → ...ixtures/plain/app/unhandled/SystemExit.rb
diff --git a/features/plain_handled_errors.feature b/features/plain_handled_errors.feature
@@ -0,0 +1,57 @@
+Feature: Plain handled errors
+
+Background:
+  Given I set environment variable "BUGSNAG_API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
+  And I set environment variable "MAZE_API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
+  And I set environment variable "APP_PATH" to "/usr/src"
+  And I configure the bugsnag endpoint
+
+Scenario Outline: A handled error sends a report
+  And I set environment variable "RUBY_VERSION" to "<ruby version>"
+  And I build the service "plain-ruby" from the compose file "features/fixtures/plain/docker-compose.yml"
+  And I run the command "plain-ruby bundle exec ruby handled/<file>.rb" on the service "features/fixtures/plain/docker-compose.yml"
+  And I wait for 1 second
+  Then I should receive a request
+  And the request used the Ruby notifier
+  And the request used payload v4 headers
+  And the request contained the api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
+  And the event "unhandled" is false
+  And the event "severity" is "warning"
+  And the event "severityReason.type" is "handledException"
+  And the exception "errorClass" equals "RuntimeError"
+  And the "file" of stack frame 0 equals "/usr/src/handled/<file>.rb"
+  And the "lineNumber" of stack frame 0 equals <lineNumber>
+
+  Examples:
+  | ruby version | file               | lineNumber |
+  | 1.9.3        | NotifyException    | 6          |
+  | 1.9.3        | NotifyString       | 8          |
+  | 2.0          | NotifyException    | 6          |
+  | 2.0          | NotifyString       | 8          |
+  | 2.1          | NotifyException    | 6          |
+  | 2.1          | NotifyString       | 8          |
+  | 2.2          | NotifyException    | 6          |
+  | 2.2          | NotifyString       | 8          |
+  | 2.3          | NotifyException    | 6          |
+  | 2.3          | NotifyString       | 8          |
+  | 2.4          | NotifyException    | 6          |
+  | 2.4          | NotifyString       | 8          |
+  | 2.5          | NotifyException    | 6          |
+  | 2.5          | NotifyString       | 8          |
+
+Scenario Outline: A handled error doesn't send a report when the :skip_bugsnag flag is set
+  And I set environment variable "ruby_version" to "<ruby version>"
+  And I build the service "plain-ruby" from the compose file "features/fixtures/plain/docker-compose.yml"
+  And I run the command "plain-ruby bundle exec ruby unhandled/IgnoreException.rb" on the service "features/fixtures/plain/docker-compose.yml"
+  And I wait for 1 second
+  Then I should receive 0 requests
+
+  Examples:
+  | ruby version |
+  | 1.9.3        |
+  | 2.0          |
+  | 2.1          |
+  | 2.2          |
+  | 2.3          |
+  | 2.4          |
+  | 2.5          |
diff --git a/features/plain_unhandled_errors.feature b/features/plain_unhandled_errors.feature
@@ -2,102 +2,105 @@ Feature: Plain unhandled errors
 
 Background:
   Given I set environment variable "BUGSNAG_API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
+  Given I set environment variable "MAZE_API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
   And I set environment variable "APP_PATH" to "/usr/src"
   And I configure the bugsnag endpoint
 
 Scenario Outline: An unhandled error sends a report
-  Given I set environment variable "MAZE_API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
   And I set environment variable "RUBY_VERSION" to "<ruby version>"
   And I build the service "plain-ruby" from the compose file "features/fixtures/plain/docker-compose.yml"
   And I run the command "plain-ruby bundle exec ruby unhandled/<file>.rb" on the service "features/fixtures/plain/docker-compose.yml"
   And I wait for 1 second
   Then I should receive a request
-  And the "Bugsnag-API-Key" header equals "a35a2a72bd230ac0aa0f52715bbdc6aa"
+  And the request used the Ruby notifier
+  And the request used payload v4 headers
+  And the request contained the api key "a35a2a72bd230ac0aa0f52715bbdc6aa"
   And the event "unhandled" is true
   And the event "severity" is "error"
+  And the event "severityReason.type" is "unhandledException"
   And the exception "errorClass" equals "<error>"
   And the "file" of stack frame 0 equals "/usr/src/unhandled/<file>.rb"
-
+  And the "lineNumber" of stack frame 0 equals <lineNumber>
+
   Examples:
-  | ruby version | file            | error          |
-  | 1.9.3        | RuntimeError    | RuntimeError   |
-  | 1.9.3        | LoadError       | LoadError      |
-  | 1.9.3        | SyntaxError     | SyntaxError    |
-  | 1.9.3        | LocalJumpError  | LocalJumpError |
-  | 1.9.3        | NameError       | NameError      |
-  | 1.9.3        | NoMethodError   | NoMethodError  |
-  | 1.9.3        | SystemCallError | Errno::ENOENT  |
-  | 1.9.3        | CustomError     | CustomError    |
-  | 2.0.0        | RuntimeError    | RuntimeError   |
-  | 2.0.0        | LoadError       | LoadError      |
-  | 2.0.0        | SyntaxError     | SyntaxError    |
-  | 2.0.0        | LocalJumpError  | LocalJumpError |
-  | 2.0.0        | NameError       | NameError      |
-  | 2.0.0        | NoMethodError   | NoMethodError  |
-  | 2.0.0        | SystemCallError | Errno::ENOENT  |
-  | 2.0.0        | CustomError     | CustomError    |
-  | 2.1.0        | RuntimeError    | RuntimeError   |
-  | 2.1.0        | LoadError       | LoadError      |
-  | 2.1.0        | SyntaxError     | SyntaxError    |
-  | 2.1.0        | LocalJumpError  | LocalJumpError |
-  | 2.1.0        | NameError       | NameError      |
-  | 2.1.0        | NoMethodError   | NoMethodError  |
-  | 2.1.0        | SystemCallError | Errno::ENOENT  |
-  | 2.1.0        | CustomError     | CustomError    |
-  | 2.2.0        | RuntimeError    | RuntimeError   |
-  | 2.2.0        | LoadError       | LoadError      |
-  | 2.2.0        | SyntaxError     | SyntaxError    |
-  | 2.2.0        | LocalJumpError  | LocalJumpError |
-  | 2.2.0        | NameError       | NameError      |
-  | 2.2.0        | NoMethodError   | NoMethodError  |
-  | 2.2.0        | SystemCallError | Errno::ENOENT  |
-  | 2.2.0        | CustomError     | CustomError    |
-  | 2.3.0        | RuntimeError    | RuntimeError   |
-  | 2.3.0        | LoadError       | LoadError      |
-  | 2.3.0        | SyntaxError     | SyntaxError    |
-  | 2.3.0        | LocalJumpError  | LocalJumpError |
-  | 2.3.0        | NameError       | NameError      |
-  | 2.3.0        | NoMethodError   | NoMethodError  |
-  | 2.3.0        | SystemCallError | Errno::ENOENT  |
-  | 2.3.0        | CustomError     | CustomError    |
-  | 2.4.0        | RuntimeError    | RuntimeError   |
-  | 2.4.0        | LoadError       | LoadError      |
-  | 2.4.0        | SyntaxError     | SyntaxError    |
-  | 2.4.0        | LocalJumpError  | LocalJumpError |
-  | 2.4.0        | NameError       | NameError      |
-  | 2.4.0        | NoMethodError   | NoMethodError  |
-  | 2.4.0        | SystemCallError | Errno::ENOENT  |
-  | 2.4.0        | CustomError     | CustomError    |
-  | 2.5.0        | RuntimeError    | RuntimeError   |
-  | 2.5.0        | LoadError       | LoadError      |
-  | 2.5.0        | SyntaxError     | SyntaxError    |
-  | 2.5.0        | LocalJumpError  | LocalJumpError |
-  | 2.5.0        | NameError       | NameError      |
-  | 2.5.0        | NoMethodError   | NoMethodError  |
-  | 2.5.0        | SystemCallError | Errno::ENOENT  |
-  | 2.5.0        | CustomError     | CustomError    |
+  | ruby version | file            | error          | lineNumber |
+  | 1.9.3        | RuntimeError    | RuntimeError   | 6          |
+  | 1.9.3        | LoadError       | LoadError      | 6          |
+  | 1.9.3        | SyntaxError     | SyntaxError    | 6          |
+  | 1.9.3        | LocalJumpError  | LocalJumpError | 7          |
+  | 1.9.3        | NameError       | NameError      | 6          |
+  | 1.9.3        | NoMethodError   | NoMethodError  | 6          |
+  | 1.9.3        | SystemCallError | Errno::ENOENT  | 6          |
+  | 1.9.3        | CustomError     | CustomError    | 9          |
+  | 2.0          | RuntimeError    | RuntimeError   | 6          |
+  | 2.0          | LoadError       | LoadError      | 6          |
+  | 2.0          | SyntaxError     | SyntaxError    | 6          |
+  | 2.0          | LocalJumpError  | LocalJumpError | 7          |
+  | 2.0          | NameError       | NameError      | 6          |
+  | 2.0          | NoMethodError   | NoMethodError  | 6          |
+  | 2.0          | SystemCallError | Errno::ENOENT  | 6          |
+  | 2.0          | CustomError     | CustomError    | 9          |
+  | 2.1          | RuntimeError    | RuntimeError   | 6          |
+  | 2.1          | LoadError       | LoadError      | 6          |
+  | 2.1          | SyntaxError     | SyntaxError    | 6          |
+  | 2.1          | LocalJumpError  | LocalJumpError | 7          |
+  | 2.1          | NameError       | NameError      | 6          |
+  | 2.1          | NoMethodError   | NoMethodError  | 6          |
+  | 2.1          | SystemCallError | Errno::ENOENT  | 6          |
+  | 2.1          | CustomError     | CustomError    | 9          |
+  | 2.2          | RuntimeError    | RuntimeError   | 6          |
+  | 2.2          | LoadError       | LoadError      | 6          |
+  | 2.2          | SyntaxError     | SyntaxError    | 6          |
+  | 2.2          | LocalJumpError  | LocalJumpError | 7          |
+  | 2.2          | NameError       | NameError      | 6          |
+  | 2.2          | NoMethodError   | NoMethodError  | 6          |
+  | 2.2          | SystemCallError | Errno::ENOENT  | 6          |
+  | 2.2          | CustomError     | CustomError    | 9          |
+  | 2.3          | RuntimeError    | RuntimeError   | 6          |
+  | 2.3          | LoadError       | LoadError      | 6          |
+  | 2.3          | SyntaxError     | SyntaxError    | 6          |
+  | 2.3          | LocalJumpError  | LocalJumpError | 7          |
+  | 2.3          | NameError       | NameError      | 6          |
+  | 2.3          | NoMethodError   | NoMethodError  | 6          |
+  | 2.3          | SystemCallError | Errno::ENOENT  | 6          |
+  | 2.3          | CustomError     | CustomError    | 9          |
+  | 2.4          | RuntimeError    | RuntimeError   | 6          |
+  | 2.4          | LoadError       | LoadError      | 6          |
+  | 2.4          | SyntaxError     | SyntaxError    | 6          |
+  | 2.4          | LocalJumpError  | LocalJumpError | 7          |
+  | 2.4          | NameError       | NameError      | 6          |
+  | 2.4          | NoMethodError   | NoMethodError  | 6          |
+  | 2.4          | SystemCallError | Errno::ENOENT  | 6          |
+  | 2.4          | CustomError     | CustomError    | 9          |
+  | 2.5          | RuntimeError    | RuntimeError   | 6          |
+  | 2.5          | LoadError       | LoadError      | 6          |
+  | 2.5          | SyntaxError     | SyntaxError    | 6          |
+  | 2.5          | LocalJumpError  | LocalJumpError | 7          |
+  | 2.5          | NameError       | NameError      | 6          |
+  | 2.5          | NoMethodError   | NoMethodError  | 6          |
+  | 2.5          | SystemCallError | Errno::ENOENT  | 6          |
+  | 2.5          | CustomError     | CustomError    | 9          |
 
-Scenario Outline: An unhandled error shouldn't send a report
-  Given I set environment variable "MAZE_API_KEY" to "a35a2a72bd230ac0aa0f52715bbdc6aa"
+Scenario Outline: An unhandled error doesn't send a report
   And I set environment variable "ruby_version" to "<ruby version>"
   And I build the service "plain-ruby" from the compose file "features/fixtures/plain/docker-compose.yml"
   And I run the command "plain-ruby bundle exec ruby unhandled/<file>.rb" on the service "features/fixtures/plain/docker-compose.yml"
   And I wait for 1 second
   Then I should receive 0 requests
-  
+
   Examples:
-  | ruby version | file        |
+  | ruby version | file         |
   | 1.9.3        | Interrupt    |
   | 1.9.3        | SystemExit   |
-  | 2.0.0        | Interrupt    |
-  | 2.0.0        | SystemExit   |
-  | 2.1.0        | Interrupt    |
-  | 2.1.0        | SystemExit   |
-  | 2.2.0        | Interrupt    |
-  | 2.2.0        | SystemExit   |
-  | 2.3.0        | Interrupt    |
-  | 2.3.0        | SystemExit   |
-  | 2.4.0        | Interrupt    |
-  | 2.4.0        | SystemExit   |
-  | 2.5.0        | Interrupt    |
-  | 2.5.0        | SystemExit   |
+  | 2.0          | Interrupt    |
+  | 2.0          | SystemExit   |
+  | 2.1          | Interrupt    |
+  | 2.1          | SystemExit   |
+  | 2.2          | Interrupt    |
+  | 2.2          | SystemExit   |
+  | 2.3          | Interrupt    |
+  | 2.3          | SystemExit   |
+  | 2.4          | Interrupt    |
+  | 2.4          | SystemExit   |
+  | 2.5          | Interrupt    |
+  | 2.5          | SystemExit   |
diff --git a/features/steps/ruby_notifier_steps.rb b/features/steps/ruby_notifier_steps.rb
@@ -47,6 +47,21 @@
   }
 end
 
+Then("the request used payload v4 headers") do
+  steps %Q{
+    Then the "bugsnag-api-key" header is not null
+    And the "bugsnag-payload-version" header equals "4.0"
+    And the "bugsnag-sent-at" header is a timestamp
+  }
+end
+
+Then("the request contained the api key {string}") do |api_key|
+  steps %Q{
+    Then the "bugsnag-api-key" header equals "#{api_key}"
+    And the payload field "apiKey" equals "#{api_key}"
+  }
+end
+
 Then("the request used the Ruby notifier") do
   steps %Q{
     Then the payload field "notifier.name" equals "Ruby Bugsnag Notifier"