HTML 5 report marks scenario as success even though one case failed

[nikowitt]

Hi there,

while using the html5 report generator, one case in my scenario fails but the whole scenario is marked as successful.

{
  "className": "com.sobis.jaf.tests.services.quartz.JAFSchedulerTest",
  "scenarios": [
    {
      "className": "com.sobis.jaf.tests.services.quartz.JAFSchedulerTest",
      "testMethodName": "check_quartz_monitoring_grid",
      "description": "check quartz monitoring grid",
      "tagIds": [],
      "pending": false,
      "explicitParameters": [
        "stateToSet"
      ],
      "derivedParameters": [
        "stateToSet",
        "value"
      ],
      "casesAsTable": true,
      "scenarioCases": [
        {
          "caseNr": 1,
          "steps": [
            {
              "name": "logged in system user",
              "words": [
                {
                  "value": "Given",
                  "isIntroWord": true
                },
                {
                  "value": "logged in system user"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 1849831
            },
            {
              "name": "all quartz jobs are paused",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "all quartz jobs are paused"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 23003
            },
            {
              "name": "dummy job is created",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "dummy job is created"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 499366875
            },
            {
              "name": "dummy job is",
              "words": [
                {
                  "value": "When",
                  "isIntroWord": true
                },
                {
                  "value": "dummy job is"
                },
                {
                  "value": "DISABLED",
                  "argumentInfo": {
                    "parameterName": "stateToSet",
                    "argumentName": "state"
                  }
                }
              ],
              "status": "PASSED",
              "durationInNanos": 25480
            },
            {
              "name": "monitoring grid is rendered",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "monitoring grid is rendered"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 20103240
            },
            {
              "name": "json entries $ found",
              "words": [
                {
                  "value": "Then",
                  "isIntroWord": true
                },
                {
                  "value": "json entries"
                },
                {
                  "value": "true",
                  "argumentInfo": {
                    "argumentName": "exists",
                    "formattedValue": "are"
                  }
                },
                {
                  "value": "found"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 15926
            },
            {
              "name": "json result $ value",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "json result"
                },
                {
                  "value": "true",
                  "argumentInfo": {
                    "argumentName": "isAllowed",
                    "formattedValue": "contains"
                  }
                },
                {
                  "value": "value"
                },
                {
                  "value": "Disabled",
                  "argumentInfo": {
                    "parameterName": "value",
                    "argumentName": "value",
                    "formattedValue": "\"Disabled\""
                  }
                }
              ],
              "status": "PASSED",
              "durationInNanos": 2589480
            }
          ],
          "explicitArguments": [
            "DISABLED"
          ],
          "derivedArguments": [
            "DISABLED",
            "\"Disabled\""
          ],
          "success": true,
          "durationInNanos": 1260122576
        },
        {
          "caseNr": 2,
          "steps": [
            {
              "name": "logged in system user",
              "words": [
                {
                  "value": "Given",
                  "isIntroWord": true
                },
                {
                  "value": "logged in system user"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 23712
            },
            {
              "name": "all quartz jobs are paused",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "all quartz jobs are paused"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 23357
            },
            {
              "name": "dummy job is created",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "dummy job is created"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 89165944
            },
            {
              "name": "dummy job is",
              "words": [
                {
                  "value": "When",
                  "isIntroWord": true
                },
                {
                  "value": "dummy job is"
                },
                {
                  "value": "WAITING",
                  "argumentInfo": {
                    "parameterName": "stateToSet",
                    "argumentName": "state"
                  }
                }
              ],
              "status": "PASSED",
              "durationInNanos": 24419
            },
            {
              "name": "monitoring grid is rendered",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "monitoring grid is rendered"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 5154187
            },
            {
              "name": "json entries $ found",
              "words": [
                {
                  "value": "Then",
                  "isIntroWord": true
                },
                {
                  "value": "json entries"
                },
                {
                  "value": "true",
                  "argumentInfo": {
                    "argumentName": "exists",
                    "formattedValue": "are"
                  }
                },
                {
                  "value": "found"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 16633
            },
            {
              "name": "json result $ value",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "json result"
                },
                {
                  "value": "true",
                  "argumentInfo": {
                    "argumentName": "isAllowed",
                    "formattedValue": "contains"
                  }
                },
                {
                  "value": "value"
                },
                {
                  "value": "Waiting",
                  "argumentInfo": {
                    "parameterName": "value",
                    "argumentName": "value",
                    "formattedValue": "\"Waiting\""
                  }
                }
              ],
              "status": "PASSED",
              "durationInNanos": 509969
            }
          ],
          "explicitArguments": [
            "WAITING"
          ],
          "derivedArguments": [
            "WAITING",
            "\"Waiting\""
          ],
          "success": true,
          "durationInNanos": 185866401
        },
        {
          "caseNr": 3,
          "steps": [
            {
              "name": "logged in system user",
              "words": [
                {
                  "value": "Given",
                  "isIntroWord": true
                },
                {
                  "value": "logged in system user"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 22650
            },
            {
              "name": "all quartz jobs are paused",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "all quartz jobs are paused"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 16280
            },
            {
              "name": "dummy job is created",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "dummy job is created"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 124740592
            },
            {
              "name": "dummy job is",
              "words": [
                {
                  "value": "When",
                  "isIntroWord": true
                },
                {
                  "value": "dummy job is"
                },
                {
                  "value": "RUNNING",
                  "argumentInfo": {
                    "parameterName": "stateToSet",
                    "argumentName": "state"
                  }
                }
              ],
              "status": "FAILED",
              "durationInNanos": 9795573
            },
            {
              "name": "monitoring grid is rendered",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "monitoring grid is rendered"
                }
              ],
              "status": "SKIPPED",
              "durationInNanos": 0
            },
            {
              "name": "json entries $ found",
              "words": [
                {
                  "value": "Then",
                  "isIntroWord": true
                },
                {
                  "value": "json entries"
                },
                {
                  "value": "true",
                  "argumentInfo": {
                    "argumentName": "exists",
                    "formattedValue": "are"
                  }
                },
                {
                  "value": "found"
                }
              ],
              "status": "SKIPPED",
              "durationInNanos": 0
            },
            {
              "name": "json result $ value",
              "words": [
                {
                  "value": "and",
                  "isIntroWord": true
                },
                {
                  "value": "json result"
                },
                {
                  "value": "true",
                  "argumentInfo": {
                    "argumentName": "isAllowed",
                    "formattedValue": "contains"
                  }
                },
                {
                  "value": "value"
                },
                {
                  "value": "Running",
                  "argumentInfo": {
                    "parameterName": "value",
                    "argumentName": "value",
                    "formattedValue": "\"Running\""
                  }
                }
              ],
              "status": "SKIPPED",
              "durationInNanos": 0
            }
          ],
          "explicitArguments": [
            "RUNNING"
          ],
          "derivedArguments": [
            "RUNNING",
            "\"Running\""
          ],
          "success": false,
          "errorMessage": "expected:\u003c1\u003e but was:\u003c0\u003e",
          "durationInNanos": 247460066
        }
      ],
      "durationInNanos": 1260122576,
      "executionStatus": "SUCCESS"
    },
    {
      "className": "com.sobis.jaf.tests.services.quartz.JAFSchedulerTest",
      "testMethodName": "check_quartz_scheduler_global_group_pause",
      "description": "check quartz scheduler global group pause",
      "tagIds": [],
      "pending": false,
      "explicitParameters": [],
      "derivedParameters": [],
      "casesAsTable": false,
      "scenarioCases": [
        {
          "caseNr": 1,
          "steps": [
            {
              "name": "logged in user",
              "words": [
                {
                  "value": "Given",
                  "isIntroWord": true
                },
                {
                  "value": "logged in user"
                },
                {
                  "value": "monitor",
                  "argumentInfo": {
                    "argumentName": "userName",
                    "formattedValue": "\"monitor\""
                  }
                }
              ],
              "status": "PASSED",
              "durationInNanos": 40477212
            },
            {
              "name": "global pause flag is $",
              "words": [
                {
                  "value": "When",
                  "isIntroWord": true
                },
                {
                  "value": "global pause flag is"
                },
                {
                  "value": "true",
                  "argumentInfo": {
                    "argumentName": "isPaused",
                    "formattedValue": "set"
                  }
                }
              ],
              "status": "PASSED",
              "durationInNanos": 7286783
            },
            {
              "name": "pause notification $ returned",
              "words": [
                {
                  "value": "Then",
                  "isIntroWord": true
                },
                {
                  "value": "pause notification"
                },
                {
                  "value": "true",
                  "argumentInfo": {
                    "argumentName": "isPaused",
                    "formattedValue": "is"
                  }
                },
                {
                  "value": "returned"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 4857974
            },
            {
              "name": "global pause flag is $",
              "words": [
                {
                  "value": "When",
                  "isIntroWord": true
                },
                {
                  "value": "global pause flag is"
                },
                {
                  "value": "false",
                  "argumentInfo": {
                    "argumentName": "isPaused",
                    "formattedValue": "removed"
                  }
                }
              ],
              "status": "PASSED",
              "durationInNanos": 9690112
            },
            {
              "name": "pause notification $ returned",
              "words": [
                {
                  "value": "Then",
                  "isIntroWord": true
                },
                {
                  "value": "pause notification"
                },
                {
                  "value": "false",
                  "argumentInfo": {
                    "argumentName": "isPaused",
                    "formattedValue": "is not"
                  }
                },
                {
                  "value": "returned"
                }
              ],
              "status": "PASSED",
              "durationInNanos": 1921672
            }
          ],
          "explicitArguments": [],
          "derivedArguments": [],
          "success": true,
          "durationInNanos": 71544954
        }
      ],
      "durationInNanos": 71544954,
      "executionStatus": "SUCCESS"
    }
  ],
  "tagMap": {}
}

When dummy job is running, status is "FAILED", but the whole execution status is SUCCESS. Am I doing something wrong?

The test case that fails:

    @Test
    // PIRSJAVA-6172
    @DataProvider({ "DISABLED", "WAITING", "RUNNING" })
    public void check_quartz_monitoring_grid(JAFScheduler.JAFSchedulerTriggerState stateToSet) throws Exception {
        given().logged_in_system_user().and().all_quartz_jobs_are_paused().and().dummy_job_is_created();
        when().dummy_job_is(stateToSet)
                .and().monitoring_grid_is_rendered();
        then().json_entries_$_found(true)
                .and()
                .json_result_$_value(true, WordUtils.capitalizeFully(stateToSet.toString()));
    }

Best regards,
Niko

[janschaefer]

That is strange. It even seems that this not even a bug in the HTML reporter but rather JGiven Core as the JSON is already wrong.

[nikowitt]

If you need additional code snippets, please let me know.