diff --git a/api/src/main/java/com/technophobia/substeps/execution/ExecutionNodeResult.java b/api/src/main/java/com/technophobia/substeps/execution/ExecutionNodeResult.java index 7395a44e..f6e05db0 100644 --- a/api/src/main/java/com/technophobia/substeps/execution/ExecutionNodeResult.java +++ b/api/src/main/java/com/technophobia/substeps/execution/ExecutionNodeResult.java @@ -134,7 +134,14 @@ public void setFailure(SubstepExecutionFailure substepExecutionFailure) { EnumSet excluded = EnumSet.of(ExecutionResult.PARSE_FAILURE, ExecutionResult.SETUP_TEARDOWN_FAILURE); if (!excluded.contains(this.result)) { - this.result = ExecutionResult.FAILED; + + if(substepExecutionFailure.isNonCritical()){ + this.result = ExecutionResult.NON_CRITICAL_FAILURE; + } + else { + this.result = ExecutionResult.FAILED; + } + } this.substepExecutionFailure = substepExecutionFailure; diff --git a/api/src/main/java/com/technophobia/substeps/execution/node/FeatureNode.java b/api/src/main/java/com/technophobia/substeps/execution/node/FeatureNode.java index 82398793..a5590e1d 100644 --- a/api/src/main/java/com/technophobia/substeps/execution/node/FeatureNode.java +++ b/api/src/main/java/com/technophobia/substeps/execution/node/FeatureNode.java @@ -73,6 +73,37 @@ public Set getTags() { return tags; } + + + public static boolean hasNonCriticalFailure(ExecutionNode node) { + + // TODO + boolean rtn = false; + + if (node.getResult().getResult() == ExecutionResult.CHILD_FAILED){ + // there is a failure at a parent level, is the cause critical ? + // any child nodes with failures that are critical ? + + if (node instanceof NodeWithChildren){ + List children = ((NodeWithChildren) node).getChildren(); + + for (ExecutionNode child : children){ + rtn = hasNonCriticalFailure(child); + if (rtn){ + break; + } + } + } + + } + else if (node.getResult().getResult() == ExecutionResult.NON_CRITICAL_FAILURE) { + // real failure, is it critical ? + rtn = node.getResult().getFailure().isNonCritical(); + } + return rtn; + } + + public static boolean hasCriticalFailure(ExecutionNode node) { boolean rtn = false; @@ -98,4 +129,5 @@ else if (node.getResult().getResult() == ExecutionResult.FAILED) { } return rtn; } + } diff --git a/api/src/main/java/com/technophobia/substeps/execution/node/RootNode.java b/api/src/main/java/com/technophobia/substeps/execution/node/RootNode.java index e213f6d0..db9014f9 100644 --- a/api/src/main/java/com/technophobia/substeps/execution/node/RootNode.java +++ b/api/src/main/java/com/technophobia/substeps/execution/node/RootNode.java @@ -19,6 +19,7 @@ package com.technophobia.substeps.execution.node; import com.google.common.collect.Lists; +import com.technophobia.substeps.execution.ExecutionNodeResult; import com.technophobia.substeps.execution.ExecutionNodeVisitor; import java.util.List; @@ -45,6 +46,11 @@ public RootNode(String featureSetDescription, List features, String this.nonFatalTags = nonFatalTags; } + @Override + public ExecutionNodeResult getResult() { + return super.getResult(); + } + @Override public RETURN_TYPE dispatch(ExecutionNodeVisitor executionNodeVisitor) { diff --git a/api/src/main/java/com/technophobia/substeps/runner/BuildFailureManager.java b/api/src/main/java/com/technophobia/substeps/runner/BuildFailureManager.java index cd1cdbb3..9dec3a31 100644 --- a/api/src/main/java/com/technophobia/substeps/runner/BuildFailureManager.java +++ b/api/src/main/java/com/technophobia/substeps/runner/BuildFailureManager.java @@ -22,6 +22,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.technophobia.substeps.execution.AbstractExecutionNodeVisitor; +import com.technophobia.substeps.execution.ExecutionResult; import com.technophobia.substeps.execution.node.IExecutionNode; import com.technophobia.substeps.execution.node.NodeWithChildren; import com.technophobia.substeps.execution.node.RootNode; @@ -106,7 +107,7 @@ private void addFailuresToLists(IExecutionNode node, List parent List path = Lists.newArrayList(parents); path.add(node); - if (node.getResult().getResult().isFailure()) { // nodes in a state of child_failed won't have the actual failure : node.getResult().getFailure() + if (node.getResult().getResult().isFailure() || node.getResult().getResult() == ExecutionResult.NON_CRITICAL_FAILURE) { // nodes in a state of child_failed won't have the actual failure : node.getResult().getFailure() // child first if (node instanceof NodeWithChildren) { diff --git a/api/src/main/java/com/technophobia/substeps/runner/SubstepExecutionFailure.java b/api/src/main/java/com/technophobia/substeps/runner/SubstepExecutionFailure.java index 6c734f24..71c10f13 100644 --- a/api/src/main/java/com/technophobia/substeps/runner/SubstepExecutionFailure.java +++ b/api/src/main/java/com/technophobia/substeps/runner/SubstepExecutionFailure.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.technophobia.substeps.execution.ExecutionResult; import com.technophobia.substeps.execution.node.IExecutionNode; +import com.technophobia.substeps.execution.node.StepImplementationNode; import java.io.Serializable; @@ -52,6 +53,47 @@ public Long apply(final SubstepExecutionFailure failure) { private final ThrowableInfo throwableInfo; + private SubstepExecutionFailure(Throwable cause, IExecutionNode executionNode, boolean setupOrTearDown, boolean nonCritical, byte[] screenshot) { + this.cause = cause; + this.executionNode = executionNode; + this.setupOrTearDown = setupOrTearDown; + this.nonCritical = nonCritical; + this.screenshot = screenshot; + this.throwableInfo = new ThrowableInfo(cause); + } + + + + + public static SubstepExecutionFailure criticalFailure(Throwable cause, IExecutionNode node, byte[] screenshotBytes) { + SubstepExecutionFailure sef = new SubstepExecutionFailure(cause, node, false, false, screenshotBytes); + + node.getResult().setFailure(sef); + + return sef; + + } + + + public static SubstepExecutionFailure nonCriticalFailure(final Throwable cause, final IExecutionNode node, byte[] screenshotBytes) { + + SubstepExecutionFailure sef = new SubstepExecutionFailure(cause, node, false, true, screenshotBytes); + + node.getResult().setFailure(sef); + + return sef; + } + + /* + * @param cause the cause of the failure + * @param node the node that failed execution + * @param screenshot the bytes representing the screenshot taken when the node execution failed + */ +// public SubstepExecutionFailure(final Throwable cause, final IExecutionNode node, final byte[] screenshot) { +// this(cause, node); +// this.setScreenshot(screenshot); +// } + public SubstepExecutionFailure(final Throwable cause) { @@ -74,16 +116,6 @@ public SubstepExecutionFailure(final Throwable cause, final IExecutionNode node) } - /** - * @param cause the cause of the failure - * @param node the node that failed execution - * @param screenshot the bytes representing the screenshot taken when the node execution failed - */ - public SubstepExecutionFailure(final Throwable cause, final IExecutionNode node, final byte[] screenshot) { - this(cause, node); - this.setScreenshot(screenshot); - } - /** * @param cause the cause of the failure @@ -106,10 +138,16 @@ public SubstepExecutionFailure(final Throwable cause, final IExecutionNode node, node.getResult().setResult(result); } + + public static void setResult(final Throwable cause, final IExecutionNode node, final ExecutionResult result) { - final SubstepExecutionFailure sef = new SubstepExecutionFailure(cause); - sef.executionNode = node; - sef.executionNode.getResult().setFailure(sef); + + + final SubstepExecutionFailure sef = SubstepExecutionFailure.criticalFailure(cause, node, null); + + +// sef.executionNode = node; +// sef.executionNode.getResult().setFailure(sef); node.getResult().setResult(result); } @@ -184,4 +222,5 @@ public byte[] getScreenshot() { public void setScreenshot(final byte[] screenshot) { this.screenshot = screenshot; } + } diff --git a/core/src/main/java/com/technophobia/substeps/execution/node/RootNodeExecutionContext.java b/core/src/main/java/com/technophobia/substeps/execution/node/RootNodeExecutionContext.java index fde38cd9..c9b7355e 100644 --- a/core/src/main/java/com/technophobia/substeps/execution/node/RootNodeExecutionContext.java +++ b/core/src/main/java/com/technophobia/substeps/execution/node/RootNodeExecutionContext.java @@ -63,15 +63,22 @@ public SetupAndTearDown getSetupAndTeardown() { return setupAndTeardown; } + + + public boolean isNodeFailureNonCritical(final IExecutionNode node) { + + return this.nonFatalTagmanager != null && nonFatalTagmanager.isApplicable(node); + } + public void addFailure(final SubstepExecutionFailure failure) { failures.add(failure); logFailure(failure); // set the criticality of this failure - - if (!failure.isSetupOrTearDown() && this.nonFatalTagmanager != null - && nonFatalTagmanager.isApplicable(failure.getExeccutionNode())) { + if (!failure.isSetupOrTearDown() && isNodeFailureNonCritical(failure.getExeccutionNode())) { +// if (!failure.isSetupOrTearDown() && this.nonFatalTagmanager != null +// && nonFatalTagmanager.isApplicable(failure.getExeccutionNode())) { failure.setNonCritical(true); } diff --git a/core/src/main/java/com/technophobia/substeps/runner/TagManager.java b/core/src/main/java/com/technophobia/substeps/runner/TagManager.java index a3849a49..20017840 100644 --- a/core/src/main/java/com/technophobia/substeps/runner/TagManager.java +++ b/core/src/main/java/com/technophobia/substeps/runner/TagManager.java @@ -43,6 +43,15 @@ public class TagManager extends AbstractExecutionNodeVisitor { private Set acceptedTags = null; private Set excludedTags = null; + public static TagManager fromTags(final String tags){ + if (tags != null){ + return new TagManager(tags); + } + else { + return null; + } + } + public TagManager(final String tagList) { acceptedTags = new HashSet(); diff --git a/core/src/main/java/com/technophobia/substeps/runner/node/RootNodeRunner.java b/core/src/main/java/com/technophobia/substeps/runner/node/RootNodeRunner.java index 2fa844fc..cd474f21 100644 --- a/core/src/main/java/com/technophobia/substeps/runner/node/RootNodeRunner.java +++ b/core/src/main/java/com/technophobia/substeps/runner/node/RootNodeRunner.java @@ -95,6 +95,14 @@ protected void recordResult(final RootNode node, final boolean success, final Ro rootNodeStateSet = true; break; } + + if (FeatureNode.hasNonCriticalFailure(featureNode)){ + SubstepsRuntimeException e = new SubstepsRuntimeException("At least one NON critical Feature failed"); + SubstepExecutionFailure.setResult(e, node, ExecutionResult.NON_CRITICAL_FAILURE); + + context.getNotificationDistributor().onNodeFailed(node, e); + rootNodeStateSet = true; + } } if (!rootNodeStateSet) { diff --git a/core/src/main/java/com/technophobia/substeps/runner/node/StepImplementationNodeRunner.java b/core/src/main/java/com/technophobia/substeps/runner/node/StepImplementationNodeRunner.java index 1b59700a..8bbe9f92 100644 --- a/core/src/main/java/com/technophobia/substeps/runner/node/StepImplementationNodeRunner.java +++ b/core/src/main/java/com/technophobia/substeps/runner/node/StepImplementationNodeRunner.java @@ -55,7 +55,16 @@ protected boolean execute(StepImplementationNode node, RootNodeExecutionContext private void addFailure(StepImplementationNode node, RootNodeExecutionContext context, Throwable t) { byte[] screenshotBytes = attemptScreenshot(node, context); - context.addFailure(new SubstepExecutionFailure(t, node, screenshotBytes)); + + SubstepExecutionFailure failure; + if (context.isNodeFailureNonCritical(node)){ + failure = SubstepExecutionFailure.nonCriticalFailure(t, node, screenshotBytes); + } + else { + failure = SubstepExecutionFailure.criticalFailure(t,node, screenshotBytes); + } + + context.addFailure(failure); } @Override diff --git a/core/src/main/resources/static/css/substeps.css b/core/src/main/resources/static/css/substeps.css index 7ce69959..cab49d81 100644 --- a/core/src/main/resources/static/css/substeps.css +++ b/core/src/main/resources/static/css/substeps.css @@ -133,3 +133,11 @@ i.jstree-icon.jstree-themeicon.PARSE_FAILURE.jstree-themeicon-custom background-size: auto; background-position: center center; } + +i.jstree-icon.jstree-themeicon.NON_CRITICAL_FAILURE.jstree-themeicon-custom +{ + background-image: url("../img/NON_CRITICAL_FAILURE.png"); + background-size: auto; + background-position: center center; +} + diff --git a/core/src/main/scala/org/substeps/report/ExecutionResultsCollector.scala b/core/src/main/scala/org/substeps/report/ExecutionResultsCollector.scala index d024bd7b..314e0233 100644 --- a/core/src/main/scala/org/substeps/report/ExecutionResultsCollector.scala +++ b/core/src/main/scala/org/substeps/report/ExecutionResultsCollector.scala @@ -267,8 +267,10 @@ class ExecutionResultsCollector extends IExecutionResultsCollector { }) + val result = if (Option(featureNode.getResult.getFailure).isDefined && featureNode.getResult.getFailure.isNonCritical) "NON_CRITICAL_FAILURE" else featureNode.getResult.getResult.name() + val data = - FeatureSummary("FeatureNode", featureNode.getFilename, featureNode.getResult.getResult.name(), featureNode.getId, + FeatureSummary("FeatureNode", featureNode.getFilename, result, featureNode.getId, Some(featureNode.getResult.getRunningDuration), featureNode.getDescription, scenarios.toList, featureNode.getTags.toList) @@ -299,7 +301,11 @@ class ExecutionResultsCollector extends IExecutionResultsCollector { case _ => None } - // TODO - add this into the node detail.. + Option(result.getFailure).map(sef => { + + log.info(" *** got Substep Exec Failure: critical : " + sef.isNonCritical) + + }) val data = diff --git a/core/src/main/scala/org/substeps/report/NodeDetail.scala b/core/src/main/scala/org/substeps/report/NodeDetail.scala index 33a18704..65d6310c 100644 --- a/core/src/main/scala/org/substeps/report/NodeDetail.scala +++ b/core/src/main/scala/org/substeps/report/NodeDetail.scala @@ -27,7 +27,10 @@ case object NodeDetail { val stackTrace = scenarioNode.getResult.getFailure.getCause.getStackTrace.toList.map(elem => elem.toString) - NodeDetail("BasicScenarioNode", scenarioNode.getParent.getFilename, scenarioNode.getLineNumber, scenarioNode.getResult.getResult.name(), scenarioNode.getId.toInt, + val result = if (scenarioNode.getResult.getFailure.isNonCritical) "NON_CRITICAL_FAILURE" else scenarioNode.getResult.getResult.name() + + + NodeDetail("BasicScenarioNode", scenarioNode.getParent.getFilename, scenarioNode.getLineNumber, result, scenarioNode.getId.toInt, Option(scenarioNode.getResult.getRunningDuration), scenarioNode.getDescription, None, children, Some(scenarioNode.getResult.getFailure.getCause.getMessage), Some(stackTrace), None, tags, None, None) diff --git a/core/src/test/java/com/technophobia/substeps/runner/ExecutionNodeRunnerTest.java b/core/src/test/java/com/technophobia/substeps/runner/ExecutionNodeRunnerTest.java index d70f7039..5967cc43 100644 --- a/core/src/test/java/com/technophobia/substeps/runner/ExecutionNodeRunnerTest.java +++ b/core/src/test/java/com/technophobia/substeps/runner/ExecutionNodeRunnerTest.java @@ -788,164 +788,7 @@ public void testNonCriticalFailures() { } - - @Test - public void testValidErrorPlusNonCriticalFailures() throws NoSuchFieldException, IllegalAccessException { - - // three features, a scenario each, 1 fails (crit), 1 fails(non crit), 1 pass. mixup the ordering - - String[] critNonCritPass = {"scenario crit fail", "scenario non crit fail", "scenario pass"}; - String[] nonCritCritPass = {"scenario non crit fail", "scenario crit fail", "scenario pass"}; - String[] passCritNonCrit = {"scenario pass", "scenario crit fail", "scenario non crit fail"}; - - - String[][] scenarios = {nonCritCritPass, critNonCritPass, passCritNonCrit}; - - for (String[] ordering : scenarios) { - - System.out.println("\n\n** Running scenario: " + Arrays.toString(ordering)); - - final Method nonFailMethod = getNonFailMethod(); - final Method failMethod = getFailMethod(); - Assert.assertNotNull(nonFailMethod); - Assert.assertNotNull(failMethod); - - final TestRootNodeBuilder rootNodeBuilder = new TestRootNodeBuilder(); - - // featureBuilder.addTags("toRun", "canFail"); - - TestBasicScenarioNodeBuilder scenario1Builder = null; - TestBasicScenarioNodeBuilder scenario2Builder = null; - TestBasicScenarioNodeBuilder scenario3Builder = null; - - TestFeatureNodeBuilder f1Builder = null; - TestFeatureNodeBuilder f2Builder = null; - TestFeatureNodeBuilder f3Builder = null; - - for (String scenarioName : ordering) { - - if (scenarioName.equals("scenario crit fail")) { - - f1Builder = rootNodeBuilder.addFeature(new Feature("crit fail feature", "file")); - - scenario1Builder = f1Builder.addBasicScenario("scenario crit fail"); - scenario1Builder.addTags(ImmutableSet.of("toRun")); - scenario1Builder.addStepImpl(getClass(), nonFailMethod).addStepImpl(getClass(), failMethod).addStepImpl(getClass(), nonFailMethod); - } - - if (scenarioName.equals("scenario non crit fail")) { - - f2Builder = rootNodeBuilder.addFeature(new Feature("non crit fail feature", "file")); - f2Builder.addTags("canFail"); - - scenario2Builder = f2Builder.addBasicScenario("scenario non crit fail"); - scenario2Builder.addStepImpl(getClass(), nonFailMethod).addStepImpl(getClass(), failMethod).addStepImpl(getClass(), nonFailMethod); - scenario2Builder.addTags(ImmutableSet.of("toRun", "canFail")); - } - - if (scenarioName.equals("scenario pass")) { - f3Builder = rootNodeBuilder.addFeature(new Feature("pass feature", "file")); - - scenario3Builder = f3Builder.addBasicScenario("scenario pass"); - scenario3Builder.addStepImpl(getClass(), nonFailMethod).addStepImpl(getClass(), nonFailMethod).addStepImpl(getClass(), nonFailMethod); - scenario3Builder.addTags(ImmutableSet.of("toRun")); - } - } - - final RootNode rootNode = rootNodeBuilder.build(); - - BasicScenarioNode scenario1 = scenario1Builder.getBuilt(); - BasicScenarioNode scenario2 = scenario2Builder.getBuilt(); - BasicScenarioNode scenario3 = scenario3Builder.getBuilt(); - - final INotificationDistributor notificationDistributor = getPrivateField(runner, "notificationDistributor"); - final SetupAndTearDown setupAndTearDown = mock(SetupAndTearDown.class); - - TagManager nonFatalTagManager = new TagManager("canFail"); - - final RootNodeExecutionContext nodeExecutionContext = new RootNodeExecutionContext(notificationDistributor, - Lists.newArrayList(), setupAndTearDown, nonFatalTagManager, new ImplementationCache()); - - setPrivateField(runner, "rootNode", rootNode); - setPrivateField(runner, "nodeExecutionContext", nodeExecutionContext); - - runner.run(); - - BuildFailureManager bfm = new BuildFailureManager(); - - Field criticalFailuresField = BuildFailureManager.class.getDeclaredField("criticalFailures"); - criticalFailuresField.setAccessible(true); - List> criticalFailures = (List>) criticalFailuresField.get(bfm); - - Field nonCriticalFailuresField = BuildFailureManager.class.getDeclaredField("nonCriticalFailures"); - nonCriticalFailuresField.setAccessible(true); - List> nonCriticalFailures = (List>) nonCriticalFailuresField.get(bfm); - - - bfm.addExecutionResult(rootNode); - -// Assert.assertFalse("non critical failure incorrectly reported as critical", bfm.testSuiteFailed()); -// Assert.assertFalse("non critical failure reporting issue", bfm.testSuiteCompletelyPassed()); - - // TODO should be 4 criticals (3 for the feature, scenario, step) + 1 for the root - // cenario crit fail, scenario non crit fail, scenario pass - the last non critical plays a factor? - // TODO test out what the rootnode execution context thinks... - - System.out.println("build failure info: " + - bfm.getBuildFailureInfo()); - - - Assert.assertThat("expecting a critical failure", criticalFailures.size(), is(1)); - Assert.assertThat("expecting a non critical failure", nonCriticalFailures.size(), is(1)); - - - final List failures = runner.getFailures(); - - Assert.assertThat(rootNode.getResult().getResult(), is(ExecutionResult.FAILED)); - - Assert.assertThat(f1Builder.getBuilt().getResult().getResult(), is(ExecutionResult.CHILD_FAILED)); - Assert.assertThat(f2Builder.getBuilt().getResult().getResult(), is(ExecutionResult.CHILD_FAILED)); - - Assert.assertThat(f3Builder.getBuilt().getResult().getResult(), is(ExecutionResult.PASSED)); - - - Assert.assertThat(scenario1.getResult().getResult(), is(ExecutionResult.CHILD_FAILED)); - Assert.assertThat(scenario2.getResult().getResult(), is(ExecutionResult.CHILD_FAILED)); - Assert.assertThat(scenario3.getResult().getResult(), is(ExecutionResult.PASSED)); - - - Assert.assertThat(scenario1.getChildren().get(0).getResult().getResult(), is(ExecutionResult.PASSED)); - Assert.assertThat(scenario1.getChildren().get(1).getResult().getResult(), is(ExecutionResult.FAILED)); - Assert.assertThat(scenario1.getChildren().get(2).getResult().getResult(), is(ExecutionResult.NOT_RUN)); - - Assert.assertThat(scenario2.getChildren().get(0).getResult().getResult(), is(ExecutionResult.PASSED)); - Assert.assertThat(scenario2.getChildren().get(1).getResult().getResult(), is(ExecutionResult.FAILED)); - Assert.assertThat(scenario2.getChildren().get(2).getResult().getResult(), is(ExecutionResult.NOT_RUN)); -// -// Assert.assertThat(outlineScenarioBuilder.getBuilt().getResult().getResult(), is(ExecutionResult.FAILED)); -// Assert.assertThat(row1ScenarioBuilder.getBuilt().getChildren().get(0).getResult().getResult(), -// is(ExecutionResult.PASSED)); -// Assert.assertThat(row1ScenarioBuilder.getBuilt().getChildren().get(1).getResult().getResult(), -// is(ExecutionResult.FAILED)); -// Assert.assertThat(row1ScenarioBuilder.getBuilt().getChildren().get(2).getResult().getResult(), -// is(ExecutionResult.NOT_RUN)); -// -// Assert.assertThat(row2ScenarioBuilder.getBuilt().getChildren().get(0).getResult().getResult(), -// is(ExecutionResult.PASSED)); -// Assert.assertThat(row2ScenarioBuilder.getBuilt().getChildren().get(1).getResult().getResult(), -// is(ExecutionResult.PASSED)); -// Assert.assertThat(row2ScenarioBuilder.getBuilt().getChildren().get(2).getResult().getResult(), -// is(ExecutionResult.PASSED)); - - // TODO check that the number of errors is correct - - Assert.assertFalse("expecting some failures", failures.isEmpty()); - - // just two failures for the actual steps that failed - Assert.assertThat(failures.size(), is(2)); - } - } - + @Test public void testExecutionNodeTreeBuildingWithScenarioName() { diff --git a/core/src/test/scala/org/substeps/report/ReportBuilderTest.scala b/core/src/test/scala/org/substeps/report/ReportBuilderTest.scala index a8f41708..e5aac94d 100644 --- a/core/src/test/scala/org/substeps/report/ReportBuilderTest.scala +++ b/core/src/test/scala/org/substeps/report/ReportBuilderTest.scala @@ -21,11 +21,19 @@ class ReportBuilderTest extends FlatSpec with ShouldMatchers{ val UTF8 = Charset.forName("UTF-8") + def getOutputDir = { + val now: LocalDateTime = LocalDateTime.now + val f = new File("target/substeps-report_" + now.format(DateTimeFormatter.ofPattern("YYYYMMddHHmmssSSS"))) + f.mkdir() + f + } + + "ReportBuilder" should "read a correct model from the source data dir" in { val now: LocalDateTime = LocalDateTime.now - val outputDir = new File("target/substeps-report_" + now.format(DateTimeFormatter.ofPattern("YYYYMMddHHmm"))) + val outputDir = getOutputDir val reportBuilder = new ReportBuilder reportBuilder.reportDir = outputDir @@ -40,11 +48,10 @@ class ReportBuilderTest extends FlatSpec with ShouldMatchers{ } - "ReportBuilder" should "build the usage tree report from raw data input" in { - val now: LocalDateTime = LocalDateTime.now - val outputDir = new File("target/substeps-report_" + now.format(DateTimeFormatter.ofPattern("YYYYMMddHHmm"))) + + val outputDir = getOutputDir val reportBuilder = new ReportBuilder reportBuilder.reportDir = outputDir @@ -61,7 +68,7 @@ class ReportBuilderTest extends FlatSpec with ShouldMatchers{ val now: LocalDateTime = LocalDateTime.now - val outputDir = new File("target/substeps-report_" + now.format(DateTimeFormatter.ofPattern("YYYYMMddHHmm"))) + val outputDir = getOutputDir val reportBuilder = new ReportBuilder reportBuilder.reportDir = outputDir @@ -87,7 +94,7 @@ class ReportBuilderTest extends FlatSpec with ShouldMatchers{ val baselineDetailDatajs = Files.toString(new File("src/test/resources/sample_feature_report_data/detail_data.js"), UTF8) - println(s"DIFF: baseline\n\n${baselineDetailDatajs}\n\n\ntestoutput:\n\n${newDetailDatajs}") + //println(s"DIFF: baseline\n\n${baselineDetailDatajs}\n\n\ntestoutput:\n\n${newDetailDatajs}") // equalToIgnoringWhiteSpace(baselineDetailDatajs).matches(newDetailDatajs) should be (true) diff --git a/core/src/test/scala/org/substeps/runner/ParsingFromSourceTests.scala b/core/src/test/scala/org/substeps/runner/ParsingFromSourceTests.scala index b8723f49..c7c83c3a 100644 --- a/core/src/test/scala/org/substeps/runner/ParsingFromSourceTests.scala +++ b/core/src/test/scala/org/substeps/runner/ParsingFromSourceTests.scala @@ -4,7 +4,8 @@ import java.io.File import java.nio.charset.Charset import com.google.common.io.Files -import com.technophobia.substeps.execution.ImplementationCache +import com.technophobia.substeps.execution.node.{ExecutionNode, IExecutionNode} +import com.technophobia.substeps.execution.{ExecutionResult, ImplementationCache} import com.technophobia.substeps.model._ import com.technophobia.substeps.model.SubSteps.StepImplementations import com.technophobia.substeps.parser.FileContents @@ -222,6 +223,291 @@ Scenario: inline table featureFile } + "running some failing features marked as non critical" must "not fail the build and be visible" in { + + // TODO - complete this test + + val simpleFeature = + """ + | Tags: all + | Feature: a simple feature + | + | Scenario: A basic failing scenario + | PassingSubstepDef + | FailingSubstepDef + | NotRun + """.stripMargin + + val substepDef = + """ + |Define: PassingSubstepDef + | AnotherPassingStepImpl + | + |Define: FailingSubstepDef + | GenerateFailure + | + """.stripMargin + + @StepImplementations + class StepImpls () extends ProvidesScreenshot { + + @SubSteps.Step("AnotherPassingStepImpl") + def anotherPassingStepImpl() = log.debug("pass") + + @SubSteps.Step("NotRun") + def notRun() = log.debug("not run") + + @SubSteps.Step("GenerateFailure") + def generateFailure() = throw new IllegalStateException("something went wrong") + + override def getScreenshotBytes: Array[Byte] = "fake screenshot bytes".getBytes + } + + + val subStepParser: SubStepDefinitionParser = new SubStepDefinitionParser(true, new DefaultSyntaxErrorReporter) + + val substepDeffileContentsFromSource = new FileContents(substepDef.split("\n").toList.asJava, new File("temp_substep_def.substeps")) + + val parentMap = subStepParser.parseSubstepFileContents(substepDeffileContentsFromSource) + + val featureFile = createFeatureFile(simpleFeature, "simple_feature_file.feature") + + val stepImplementationClasses : List[java.lang.Class[_]] = List(classOf[StepImpls]) + + val executionConfig = new SubstepsExecutionConfig + + executionConfig.setStepImplementationClasses(stepImplementationClasses.asJava) + + executionConfig.setTags("all") + executionConfig.setNonFatalTags("all") + + val syntax: Syntax = SyntaxBuilder.buildSyntax(stepImplementationClasses.asJava, parentMap) + + val tagManager = new TagManager(executionConfig.getTags) + + val nonFatalTAgManager = TagManager.fromTags(executionConfig.getNonFatalTags) + + val parameters: TestParameters = new TestParameters(tagManager, syntax, List(featureFile).asJava) + + val cfgWrapper = new ExecutionConfigWrapper(executionConfig) + val nodeTreeBuilder: ExecutionNodeTreeBuilder = new ExecutionNodeTreeBuilder(parameters, cfgWrapper) + + // building the tree can throw critical failures if exceptions are found + val rootNode = nodeTreeBuilder.buildExecutionNodeTree("test description") + + log.debug("rootNode 1:\n" + rootNode.toDebugString) + + val executionCollector = new ExecutionResultsCollector + val dataDir = ExecutionResultsCollector.getBaseDir(new File("target")) + executionCollector.setDataDir(dataDir) + executionCollector.setPretty(true) + + executionConfig.setDataOutputDirectory(dataDir) + + val runner = new ExecutionNodeRunner() + + + runner.addNotifier(executionCollector) + + val methodExecutorToUse = new ImplementationCache() + + val setupAndTearDown: SetupAndTearDown = new SetupAndTearDown(executionConfig.getInitialisationClasses, methodExecutorToUse) + + + val rootNode2 = runner.prepareExecutionConfig(new ExecutionConfigWrapper(executionConfig), syntax, parameters, setupAndTearDown, methodExecutorToUse, nonFatalTAgManager) + + executionCollector.initOutputDirectories(rootNode2) + + log.debug("rootNode 2:\n" + rootNode2.toDebugString) + + val finalRootNode = runner.run() + + log.debug("finalRootNode :\n" + rootNode2.toDebugString) + + finalRootNode.getResult.getResult should be (ExecutionResult.NON_CRITICAL_FAILURE) + } + + + // from ExecutionNodeRunnerTest.testValidErrorPlusNonCriticalFailures + "mixed critical and non critical failures" must "be reported as such" in { + + val f1 = ("critical-failure.feature", + """ + | Tags: toRun + | Feature: crit fail feature + | Scenario: scenario crit fail + | nonFail + | Fail + | nonFail + | + """.stripMargin) + + + val f2 = ("non-critical-failure.feature", + """ + | Tags: canFail + | Feature: non crit fail feature + | + | Tags: toRun canFail + | Scenario: scenario non crit fail + | nonFail + | Fail + | nonFail + | + """.stripMargin) + + val f3 = ("pass.feature", + """ + | Feature: pass feature + | Tags: toRun + | Scenario: scenario pass + | nonFail + | nonFail + | nonFail + """.stripMargin) + + + @StepImplementations + class StepImpls () extends ProvidesScreenshot { + + @SubSteps.Step("nonFail") + def nonFailingMethod() = System.out.println("no fail") + + @SubSteps.Step("Fail") + def failingMethod() = throw new IllegalStateException("that's it, had enough") + + override def getScreenshotBytes: Array[Byte] = "fake screenshot bytes".getBytes + } + + val features = List(f1,f2,f3) + + val (finalRootNode, failures) = runFeatures("", features, List(classOf[StepImpls]), Some("toRun"), Some("canFail")) + + println("finalRootNode:\n" + finalRootNode.toDebugString) + + // some assertions + val bfm = new BuildFailureManager(); + + + + + val criticalFailuresField = classOf[BuildFailureManager].getDeclaredField("criticalFailures") + criticalFailuresField.setAccessible(true) + val criticalFailures = criticalFailuresField.get(bfm).asInstanceOf[java.util.List[java.util.List[IExecutionNode]]].asScala // (List>) + + val nonCriticalFailuresField = classOf[BuildFailureManager].getDeclaredField("nonCriticalFailures") + nonCriticalFailuresField.setAccessible(true) + val nonCriticalFailures = nonCriticalFailuresField.get(bfm).asInstanceOf[java.util.List[java.util.List[IExecutionNode]]].asScala // (List>) + + + bfm.addExecutionResult(finalRootNode) + + criticalFailures should have size(1) + nonCriticalFailures should have size(1) + + + finalRootNode.getResult().getResult() should be (ExecutionResult.FAILED) + + val featureList = finalRootNode.getChildren.asScala + + featureList(0).getResult.getResult should be (ExecutionResult.CHILD_FAILED) + featureList(1).getResult.getResult should be (ExecutionResult.CHILD_FAILED) + featureList(2).getResult.getResult should be (ExecutionResult.PASSED) + + val sc1 = featureList(0).getChildren.asScala(0) + val sc2 = featureList(1).getChildren.asScala(0) + val sc3 = featureList(2).getChildren.asScala(0) + + sc1.getResult.getResult should be (ExecutionResult.CHILD_FAILED) + sc2.getResult.getResult should be (ExecutionResult.CHILD_FAILED) + sc3.getResult.getResult should be (ExecutionResult.PASSED) + + val scenario1Results = + sc1.getChildren.asScala.toList.map(st => (st.asInstanceOf[ExecutionNode]).getResult.getResult) + + import ExecutionResult._ + + scenario1Results should be (List(PASSED, FAILED, NOT_RUN)) + + val scenario2Results = + sc2.getChildren.asScala.toList.map(st => (st.asInstanceOf[ExecutionNode]).getResult.getResult) + + scenario2Results should be (List(PASSED, NON_CRITICAL_FAILURE, NOT_RUN)) + + + failures should have size(2) + + } + + + def runFeatures(suiteDescription : String, features : List[(String, String)], + stepImplementationClasses : List[java.lang.Class[_]], + tags : Option[String], + nonFatalTags : Option[String] + ) = { + +// val subStepParser: SubStepDefinitionParser = new SubStepDefinitionParser(true, new DefaultSyntaxErrorReporter) +// +// val substepDeffileContentsFromSource = new FileContents(substepDef.split("\n").toList.asJava, new File("temp_substep_def.substeps")) +// + val parentMap = new PatternMap[ParentStep] + + val featureFileList = + features.map(f => { + createFeatureFile(f._2, f._1) + }) + + + //val stepImplementationClasses : List[java.lang.Class[_]] = List(classOf[StepImpls]) + + val executionConfig = new SubstepsExecutionConfig + + executionConfig.setStepImplementationClasses(stepImplementationClasses.asJava) + + val syntax: Syntax = SyntaxBuilder.buildSyntax(stepImplementationClasses.asJava, parentMap) + + + + val parameters: TestParameters = new TestParameters(new TagManager(tags.getOrElse(null)), syntax, featureFileList.asJava) + + val cfgWrapper = new ExecutionConfigWrapper(executionConfig) + val nodeTreeBuilder: ExecutionNodeTreeBuilder = new ExecutionNodeTreeBuilder(parameters, cfgWrapper) + + // building the tree can throw critical failures if exceptions are found + val rootNode = nodeTreeBuilder.buildExecutionNodeTree(suiteDescription) + + log.debug("node tree builder rootNode 1:\n" + rootNode.toDebugString) + + val executionCollector = new ExecutionResultsCollector + val dataDir = ExecutionResultsCollector.getBaseDir(new File("target")) + executionCollector.setDataDir(dataDir) + executionCollector.setPretty(true) + + executionConfig.setDataOutputDirectory(dataDir) + + val runner = new ExecutionNodeRunner() + + + runner.addNotifier(executionCollector) + + val methodExecutorToUse = new ImplementationCache() + + val setupAndTearDown: SetupAndTearDown = new SetupAndTearDown(executionConfig.getInitialisationClasses, methodExecutorToUse) + + + val rootNode2 = runner.prepareExecutionConfig(new ExecutionConfigWrapper(executionConfig), syntax, parameters, setupAndTearDown, methodExecutorToUse, TagManager.fromTags(nonFatalTags.getOrElse(null))) + + executionCollector.initOutputDirectories(rootNode2) + + log.debug("prepared rootNode 2:\n" + rootNode2.toDebugString) + + val finalRootNode = runner.run() + + (finalRootNode, runner.getFailures.asScala) + + + } + /** * NB. this is the test that generates the source data, then used to test out report building @@ -343,7 +629,7 @@ Scenario: inline table executionCollector.setDataDir(dataDir) executionCollector.setPretty(true) - executionConfig.setDataOutputDirectory(dataDir.getAbsolutePath) + executionConfig.setDataOutputDirectory(dataDir) val runner = new ExecutionNodeRunner() @@ -464,7 +750,6 @@ Scenario: inline table - }