diff --git a/plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfterClasses.java b/plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfterClasses.java index b1d4c70263..81027d4cc4 100644 --- a/plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfterClasses.java +++ b/plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfterClasses.java @@ -69,24 +69,28 @@ public void evaluate() throws Throwable { errors.add(e); } - for (FrameworkMethod each : afterClasses) { - try { - frameworkMethod = each; - frameworkMethod.invokeExplosively(target); - } catch (Throwable e) { - if (ScreenshotCapturer.shouldCaptureScreenshotOnException(e)) { - if (isClassLevel()) { - createScreenshot("AfterClass"); - } else { - createScreenshot("After"); - } + try { + for (FrameworkMethod each : afterClasses) { + + frameworkMethod = each; + frameworkMethod.invokeExplosively(target); + } + + // Should be run before testing the failures in this context + requirements.runAfterClass(); + + } catch (Throwable e) { + if (ScreenshotCapturer.shouldCaptureScreenshotOnException(e)) { + if (isClassLevel()) { + createScreenshot("AfterClass"); + } else { + createScreenshot("After"); } - errors.add(e); } + errors.add(e); } MultipleFailureException.assertEmpty(errors); - - requirements.runAfterClass(); + } } diff --git a/plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfters.java b/plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfters.java index b49b513f55..124c7675e1 100644 --- a/plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfters.java +++ b/plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfters.java @@ -72,25 +72,28 @@ public void evaluate() throws Throwable { } catch (Throwable e) { errors.add(e); } - - for (FrameworkMethod each : fAfters) { - try { - frameworkMethod = each; - frameworkMethod.invokeExplosively(target); - } catch (Throwable e) { - if (ScreenshotCapturer.shouldCaptureScreenshotOnException(e)) { - if (isClassLevel()) { - createScreenshot("AfterClass"); - } else { - createScreenshot("After"); - } + + try { + for (FrameworkMethod each : fAfters) { + frameworkMethod = each; + frameworkMethod.invokeExplosively(target); + } + + // Should be run before testing the failures in tests + requirements.runAfter(); + + } catch (Throwable e) { + if (ScreenshotCapturer.shouldCaptureScreenshotOnException(e)) { + if (isClassLevel()) { + createScreenshot("AfterClass"); + } else { + createScreenshot("After"); } - errors.add(e); } + errors.add(e); } MultipleFailureException.assertEmpty(errors); - - requirements.runAfter(); + } } diff --git a/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/injection/InjectRequirementsTest.java b/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/injection/InjectRequirementsTest.java index 9a0c986540..82ce5901d9 100644 --- a/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/injection/InjectRequirementsTest.java +++ b/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/injection/InjectRequirementsTest.java @@ -44,7 +44,7 @@ public static void beforeClass(){ previousStaticRequirement = staticRequirement; run++; } else { - assertNotEquals("Static requirement should be injected for every test before @BeboreClass method", + assertNotEquals("Static requirement should be injected for every test before @BeforeClass method", previousStaticRequirement, staticRequirement); } } diff --git a/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/OrderRunnerRequirement.java b/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/OrderRunnerRequirement.java new file mode 100644 index 0000000000..68809f37db --- /dev/null +++ b/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/OrderRunnerRequirement.java @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2018 Red Hat, Inc and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat Inc. - initial API and implementation + *******************************************************************************/ +package org.eclipse.reddeer.junit.test.integration.runner.order; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.eclipse.reddeer.junit.requirement.Requirement; +import org.eclipse.reddeer.junit.test.integration.runner.order.OrderRunnerRequirement.RequirementOrderAnnotation; + +public class OrderRunnerRequirement implements Requirement { + + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + public @interface RequirementOrderAnnotation { + + } + + @Override + public void fulfill() { + TestSequence.addFulfill(OrderRunnerRequirement.class); + } + + @Override + public void setDeclaration(RequirementOrderAnnotation declaration) { + } + + @Override + public void cleanUp() { + TestSequence.addCleanup(OrderRunnerRequirement.class); + } + + @Override + public void runBeforeClass() { + TestSequence.addReqBeforeClass(OrderRunnerRequirement.class); + } + + @Override + public void runBefore() { + TestSequence.addReqBefore(OrderRunnerRequirement.class); + } + + @Override + public void runAfterClass() { + TestSequence.addReqAfterClass(OrderRunnerRequirement.class); + } + + @Override + public void runAfter() { + TestSequence.addReqAfter(OrderRunnerRequirement.class); + } + + @Override + public RequirementOrderAnnotation getDeclaration() { + return null; + } + +} diff --git a/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/RequirementsOrderRunnerSuite.java b/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/RequirementsOrderRunnerSuite.java new file mode 100644 index 0000000000..917d7d3dc2 --- /dev/null +++ b/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/RequirementsOrderRunnerSuite.java @@ -0,0 +1,143 @@ +/******************************************************************************* + * Copyright (c) 2018 Red Hat, Inc and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat Inc. - initial API and implementation + *******************************************************************************/ +package org.eclipse.reddeer.junit.test.integration.runner.order; + +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createAfter; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createAfterClass; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createBefore; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createBeforeClass; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createCleanup; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createFulfill; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createIAfter; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createIAfterClass; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createIBefore; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createIBeforeClass; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createReqAfter; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createReqAfterClass; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createReqBefore; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createReqBeforeClass; +import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createTest; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.reddeer.common.exception.RedDeerException; +import org.eclipse.reddeer.junit.internal.configuration.SuiteConfiguration; +import org.eclipse.reddeer.junit.test.integration.runner.IAfterTestImpl; +import org.eclipse.reddeer.junit.test.integration.runner.IBeforeTestImpl; +import org.eclipse.reddeer.junit.test.integration.runner.order.OrderRunnerRequirement.RequirementOrderAnnotation; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.RunnerBuilder; + +public class RequirementsOrderRunnerSuite extends TestSequenceRedDeerSuite { + + private static final List expectedSequence; + + static { + expectedSequence = new ArrayList(); + expectedSequence.add(createIBeforeClass(IBeforeTestImpl.class)); + expectedSequence.add(createFulfill(OrderRunnerRequirement.class)); + expectedSequence.add(createReqBeforeClass(OrderRunnerRequirement.class)); + expectedSequence.add(createBeforeClass(RequirementsOrderRunnerSuite.class)); + + expectedSequence.add(createIBefore(IBeforeTestImpl.class)); + expectedSequence.add(createReqBefore(OrderRunnerRequirement.class)); + expectedSequence.add(createBefore(RequirementsOrderRunnerSuite.class)); + expectedSequence.add(createTest(RequirementsOrderRunnerSuite.class)); + expectedSequence.add(createAfter(RequirementsOrderRunnerSuite.class)); + expectedSequence.add(createReqAfter(OrderRunnerRequirement.class)); + expectedSequence.add(createIAfter(IAfterTestImpl.class)); + + expectedSequence.add(createAfterClass(RequirementsOrderRunnerSuite.class)); + expectedSequence.add(createReqAfterClass(OrderRunnerRequirement.class)); + expectedSequence.add(createCleanup(OrderRunnerRequirement.class)); + expectedSequence.add(createIAfterClass(IAfterTestImpl.class)); + } + + @Override + protected List getExpectedSequence() { + return expectedSequence; + } + + public RequirementsOrderRunnerSuite(Class clazz, RunnerBuilder builder, + SuiteConfiguration config) throws InitializationError { + super(clazz, builder, config); + } + + public RequirementsOrderRunnerSuite(Class clazz, RunnerBuilder builder) + throws InitializationError { + super(clazz, builder); + } + + public static List getStaticExpectedSequence() { + return expectedSequence; + } + + @RunWith(RequirementsOrderRunnerSuite.class) + @RequirementOrderAnnotation + public static class RequirementOrderParentTest { + + @BeforeClass + public static void beforeClass(){ + TestSequence.addBeforeClass(RequirementsOrderRunnerSuite.class); + } + + @Before + public void before(){ + TestSequence.addBefore(RequirementsOrderRunnerSuite.class); + } + + @After + public void after(){ + TestSequence.addAfter(RequirementsOrderRunnerSuite.class); + } + + @AfterClass + public static void afterClass(){ + TestSequence.addAfterClass(RequirementsOrderRunnerSuite.class); + } + } + + public static class RequirementOrderPassedTest extends RequirementOrderParentTest { + + @Test + public void test(){ + TestSequence.addTest(RequirementsOrderRunnerSuite.class); + } + } + + public static class RequirementOrderFailedTest extends RequirementOrderParentTest { + + @Test + public void test(){ + TestSequence.addTest(RequirementsOrderRunnerSuite.class); + assertTrue("Test in failure", false); + } + } + + public static class RequirementOrderErroneousTest extends RequirementOrderParentTest { + + @Test + public void test(){ + TestSequence.addTest(RequirementsOrderRunnerSuite.class); + throw new RedDeerException("Test in error"); + } + } + + +} diff --git a/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/testcase/RequirementOrderCallWithFailsTest.java b/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/testcase/RequirementOrderCallWithFailsTest.java new file mode 100644 index 0000000000..6e5e9882d6 --- /dev/null +++ b/tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/testcase/RequirementOrderCallWithFailsTest.java @@ -0,0 +1,86 @@ +/******************************************************************************* + * Copyright (c) 2018 Red Hat, Inc and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat Inc. - initial API and implementation + *******************************************************************************/ +package org.eclipse.reddeer.junit.test.integration.runner.order.testcase; + +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.eclipse.reddeer.junit.internal.configuration.SuiteConfiguration; +import org.eclipse.reddeer.junit.runner.RedDeerSuite; +import org.eclipse.reddeer.junit.test.integration.runner.order.RequirementsOrderRunnerSuite; +import org.eclipse.reddeer.junit.test.integration.runner.order.RequirementsOrderRunnerSuite.RequirementOrderErroneousTest; +import org.eclipse.reddeer.junit.test.integration.runner.order.RequirementsOrderRunnerSuite.RequirementOrderFailedTest; +import org.eclipse.reddeer.junit.test.integration.runner.order.RequirementsOrderRunnerSuite.RequirementOrderPassedTest; +import org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runner.Runner; +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.model.InitializationError; + +/** + * Tests order of calling befores and afters of test class + * combined with requirement calling of those methods during passed/failed/erroneous tests. + * @author odockal + * + */ +@RunWith(RedDeerSuite.class) +public class RequirementOrderCallWithFailsTest { + + @Before + public void setupSequence() { + TestSequence.getRealSequence().clear(); + } + + @Test + public void testPassingTest() throws InitializationError { + SuiteConfiguration config = new SuiteConfiguration(RequirementOrderPassedTest.class); + List runners = RedDeerSuite.createSuites(RequirementOrderPassedTest.class, config); + + Runner myRunner = runners.get(0); + RunNotifier notifier = new RunNotifier(); + myRunner.run(notifier); // runner.run executes the test and static values could be retrieved + String sequenceDiff = TestSequence.diffRealSequence(RequirementsOrderRunnerSuite.getStaticExpectedSequence()); + assertTrue("Test sequence is different than expected. " + sequenceDiff , sequenceDiff == null); + } + + @Test + public void testFailingTest() throws InitializationError { + SuiteConfiguration config = new SuiteConfiguration(RequirementOrderFailedTest.class); + List runners = RedDeerSuite.createSuites(RequirementOrderFailedTest.class, config); + + Runner myRunner = runners.get(0); + RunNotifier notifier = new RunNotifier(); + myRunner.run(notifier); // runner.run executes the test and static values could be retrieved + String sequenceDiff = TestSequence.diffRealSequence(RequirementsOrderRunnerSuite.getStaticExpectedSequence()); + assertTrue("Test sequence is different than expected. " + sequenceDiff , sequenceDiff == null); + } + + @Test + public void testErroneousTest() throws InitializationError { + SuiteConfiguration config = new SuiteConfiguration(RequirementOrderErroneousTest.class); + List runners = RedDeerSuite.createSuites(RequirementOrderErroneousTest.class, config); + + Runner myRunner = runners.get(0); + RunNotifier notifier = new RunNotifier(); + myRunner.run(notifier); // runner.run executes the test and static values could be retrieved + String sequenceDiff = TestSequence.diffRealSequence(RequirementsOrderRunnerSuite.getStaticExpectedSequence()); + assertTrue("Test sequence is different than expected. " + sequenceDiff , sequenceDiff == null); + } + + @After + public void clearSequence() { + TestSequence.getRealSequence().clear(); + } +}