Skip to content

Stop execution of scenario if one step failed/inconclusive (#360) #1198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nohwnd merged 1 commit into from
Jan 5, 2019
Merged

Stop execution of scenario if one step failed/inconclusive (#360) #1198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions Functions/Gherkin.Tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,19 +369,19 @@ Describe "Check test results of steps" -Tag Gherkin {
}

It "Test result 11 is correct" {
$testResults[10] | Should -Be 'Passed'
$testResults[10] | Should -Be 'Inconclusive'
}

It "Test result 12 is correct" {
$testResults[11] | Should -Be 'Failed'
$testResults[11] | Should -Be 'Inconclusive'
}

It "Test result 13 is correct" {
$testResults[12] | Should -Be 'Inconclusive'
}

It "Test result 14 is correct" {
$testResults[13] | Should -Be 'Passed'
$testResults[13] | Should -Be 'Inconclusive'
}

It "Test result 15 is correct" {
Expand All @@ -390,7 +390,6 @@ Describe "Check test results of steps" -Tag Gherkin {

}


Describe "A generated NUnit report" -Tag Gherkin {

# Use temporary report file with Pester's test drive feature
Expand Down Expand Up @@ -523,13 +522,14 @@ Describe "A generated NUnit report" -Tag Gherkin {
Get-XmlValue "($scenario2Examples1StepsXPath/@result)[3]" | Should -Be "Success"
Get-XmlValue "($scenario2Examples1StepsXPath/@result)[4]" | Should -Be "Success"
Get-XmlValue "($scenario2Examples1StepsXPath/@result)[5]" | Should -Be "Failure"
Get-XmlValue "($scenario2Examples1StepsXPath/@result)[6]" | Should -Be "Success"
Get-XmlValue "($scenario2Examples1StepsXPath/@result)[6]" | Should -Be "Inconclusive"

Get-XmlInnerText "$scenario2Examples1StepsXPath[5]/failure/message" | Should -Be "An example error in the then clause"
if ($expectFeatureFileNameInStackTrace) {
Get-XmlInnerText "($scenario2Examples1StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*From $($expectedFeatureFileName1): line 15*"
}
Get-XmlInnerText "($scenario2Examples1StepsXPath)[5]/failure/stack-trace" | Should -BeLike "*at <ScriptBlock>, $($expectedImplementationFileName): line 23*"
Get-XmlInnerText "($scenario2Examples1StepsXPath)[6]/reason/message" | Should -Be "Step skipped (previous step did not pass)"
}

It 'should contain all steps of scenario 2 (examples 2) with correct names and test results' {
Expand Down Expand Up @@ -587,13 +587,14 @@ Describe "A generated NUnit report" -Tag Gherkin {

Get-XmlValue "($scenario4StepsXPath/@result)[1]" | Should -Be "Success"
Get-XmlValue "($scenario4StepsXPath/@result)[2]" | Should -Be "Failure"
Get-XmlValue "($scenario4StepsXPath/@result)[3]" | Should -Be "Success"
Get-XmlValue "($scenario4StepsXPath/@result)[3]" | Should -Be "Inconclusive"

Get-XmlInnerText "$scenario4StepsXPath[2]/failure/message" | Should -Be "An example error in the when clause"
if ($expectFeatureFileNameInStackTrace) {
Get-XmlInnerText "($scenario4StepsXPath)[2]/failure/stack-trace" | Should -BeLike "*From $($expectedFeatureFileName2): line 6*"
}
Get-XmlInnerText "($scenario4StepsXPath)[2]/failure/stack-trace" | Should -BeLike "*at <ScriptBlock>, $($expectedImplementationFileName): line 64*"
Get-XmlInnerText "($scenario4StepsXPath)[3]/reason/message" | Should -Be "Step skipped (previous step did not pass)"
}

It 'should contain all steps of scenario 5 (examples 1) with correct names and test results' {
Expand Down Expand Up @@ -652,8 +653,8 @@ Describe "A generated NUnit report" -Tag Gherkin {
Get-XmlValue "($scenario5Examples3cStepsXPath/@name)[3]" | Should -Be "Scenario 5 [Examples 3 3].Then step_903"

Get-XmlValue "($scenario5Examples3aStepsXPath/@result)[1]" | Should -Be "Failure"
Get-XmlValue "($scenario5Examples3aStepsXPath/@result)[2]" | Should -Be "Success"
Get-XmlValue "($scenario5Examples3aStepsXPath/@result)[3]" | Should -Be "Success"
Get-XmlValue "($scenario5Examples3aStepsXPath/@result)[2]" | Should -Be "Inconclusive"
Get-XmlValue "($scenario5Examples3aStepsXPath/@result)[3]" | Should -Be "Inconclusive"
Get-XmlValue "($scenario5Examples3bStepsXPath/@result)[1]" | Should -Be "Inconclusive"
Get-XmlValue "($scenario5Examples3bStepsXPath/@result)[2]" | Should -Be "Inconclusive"
Get-XmlValue "($scenario5Examples3bStepsXPath/@result)[3]" | Should -Be "Inconclusive"
Expand All @@ -666,6 +667,8 @@ Describe "A generated NUnit report" -Tag Gherkin {
Get-XmlInnerText "($scenario5Examples3aStepsXPath)[1]/failure/stack-trace" | Should -BeLike "*From $($expectedFeatureFileName2): line 11"
}
Get-XmlInnerText "($scenario5Examples3aStepsXPath)[1]/failure/stack-trace" | Should -BeLike "*at <ScriptBlock>, $($expectedImplementationFileName): line 71*"
Get-XmlInnerText "($scenario5Examples3aStepsXPath)[2]/reason/message" | Should -Be "Step skipped (previous step did not pass)"
Get-XmlInnerText "($scenario5Examples3aStepsXPath)[3]/reason/message" | Should -Be "Step skipped (previous step did not pass)"
Get-XmlInnerText "($scenario5Examples3bStepsXPath)[1]/reason/message" | Should -Be "Could not find implementation for step!"
Get-XmlInnerText "($scenario5Examples3bStepsXPath)[2]/reason/message" | Should -Be "Could not find implementation for step!"
Get-XmlInnerText "($scenario5Examples3bStepsXPath)[3]/reason/message" | Should -Be "Could not find implementation for step!"
Expand Down
32 changes: 26 additions & 6 deletions Functions/Gherkin.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,16 +560,18 @@ function Invoke-GherkinScenario {
New-TestDrive
Invoke-GherkinHook BeforeEachScenario $Scenario.Name $Scenario.Tags

$testResultIndexStart = $Pester.TestResult.Count

# If there's a background, run that before the test, but after hooks
if ($Background) {
foreach ($Step in $Background.Steps) {
# Run Background steps -Background so they don't output in each scenario
Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible
Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart
}
}

foreach ($Step in $Scenario.Steps) {
Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible
Invoke-GherkinStep -Step $Step -Pester $Pester -Scenario $GherkinSessionState -Visible -TestResultIndexStart $testResultIndexStart
}

Invoke-GherkinHook AfterEachScenario $Scenario.Name $Scenario.Tags
Expand Down Expand Up @@ -673,6 +675,9 @@ function Invoke-GherkinStep {

.PARAMETER ScenarioState
Gherkin state object. For internal use only

.PARAMETER TestResultIndexStart
Used to hold the test result index of the first step of the current scenario. For internal use only
#>
[CmdletBinding()]
param (
Expand All @@ -682,7 +687,9 @@ function Invoke-GherkinStep {

$Pester,

$ScenarioState
$ScenarioState,

[int] $TestResultIndexStart
)
if ($Step -is [string]) {
$KeyWord, $StepText = $Step -split "(?<=^(?:Given|When|Then|And|But))\s+"
Expand All @@ -708,10 +715,23 @@ function Invoke-GherkinStep {
}
) | & $SafeCommands["Sort-Object"] MatchCount | & $SafeCommands["Select-Object"] -First 1

if (!$StepCommand) {
$PesterErrorRecord = New-PesterErrorRecord -Result Inconclusive -Message "Could not find implementation for step!" -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText
$previousStepsNotSuccessful = $false
# Iterate over the test results of the previous steps
for ($i = $TestResultIndexStart; $i -lt ($Pester.TestResult.Count); $i++) {
$previousTestResult = $Pester.TestResult[$i].Result
if ($previousTestResult -eq "Failed" -or $previousTestResult -eq "Inconclusive") {
$previousStepsNotSuccessful = $true
break
}
}
if (!$StepCommand -or $previousStepsNotSuccessful) {
$skipMessage = if (!$StepCommand) {
"Could not find implementation for step!"
} else {
"Step skipped (previous step did not pass)"
}
$PesterErrorRecord = New-PesterErrorRecord -Result Inconclusive -Message $skipMessage -File $Step.Location.Path -Line $Step.Location.Line -LineText $DisplayText
} else {

$NamedArguments, $Parameters = Get-StepParameters $Step $StepCommand
$watch = & $SafeCommands["New-Object"] System.Diagnostics.Stopwatch
$watch.Start()
Expand Down