Fixing run after call in requirement during test failure

Signed-off-by: Ondrej Dockal <odockal@redhat.com>

plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfterClasses.java

@@ -69,24 +69,28 @@ public void evaluate() throws Throwable {
 			errors.add(e);
 		}
 
-		for (FrameworkMethod each : afterClasses) {
-			try {
-				frameworkMethod = each;
-				frameworkMethod.invokeExplosively(target);
-			} catch (Throwable e) {
-				if (ScreenshotCapturer.shouldCaptureScreenshotOnException(e)) {
-					if (isClassLevel()) {
-						createScreenshot("AfterClass");
-					} else {
-						createScreenshot("After");
-					}
+		try {
+			for (FrameworkMethod each : afterClasses) {
+
+					frameworkMethod = each;
+					frameworkMethod.invokeExplosively(target);
+			}
+
+			// Should be run before testing the failures in this context
+			requirements.runAfterClass();
+
+		} catch (Throwable e) {
+			if (ScreenshotCapturer.shouldCaptureScreenshotOnException(e)) {
+				if (isClassLevel()) {
+					createScreenshot("AfterClass");
+				} else {
+					createScreenshot("After");
 				}
-				errors.add(e);
 			}
+			errors.add(e);
 		}
 
 		MultipleFailureException.assertEmpty(errors);
-
-		requirements.runAfterClass();
+
 	}
 }

plugins/org.eclipse.reddeer.junit/src/org/eclipse/reddeer/junit/internal/runner/statement/RunAfters.java

@@ -72,25 +72,28 @@ public void evaluate() throws Throwable {
 		} catch (Throwable e) {
 			errors.add(e);
 		}
-
-		for (FrameworkMethod each : fAfters) {
-			try {
-				frameworkMethod = each;
-				frameworkMethod.invokeExplosively(target);
-			} catch (Throwable e) {
-				if (ScreenshotCapturer.shouldCaptureScreenshotOnException(e)) {
-					if (isClassLevel()) {
-						createScreenshot("AfterClass");
-					} else {
-						createScreenshot("After");
-					}
+
+		try {
+			for (FrameworkMethod each : fAfters) {
+					frameworkMethod = each;
+					frameworkMethod.invokeExplosively(target);
+			}
+
+			// Should be run before testing the failures in tests
+			requirements.runAfter();
+
+		} catch (Throwable e) {
+			if (ScreenshotCapturer.shouldCaptureScreenshotOnException(e)) {
+				if (isClassLevel()) {
+					createScreenshot("AfterClass");
+				} else {
+					createScreenshot("After");
 				}
-				errors.add(e);
 			}
+			errors.add(e);
 		}
 
 		MultipleFailureException.assertEmpty(errors);
-
-		requirements.runAfter();
+
 	}
 }

tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/injection/InjectRequirementsTest.java

@@ -44,7 +44,7 @@ public static void beforeClass(){
 			previousStaticRequirement = staticRequirement;
 			run++;
 		} else {
-			assertNotEquals("Static requirement should be injected for every test before @BeboreClass method", 
+			assertNotEquals("Static requirement should be injected for every test before @BeforeClass method", 
 					previousStaticRequirement, staticRequirement);
 		}
 	}

tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/OrderRunnerRequirement.java

@@ -0,0 +1,68 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Red Hat, Inc and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Red Hat Inc. - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.reddeer.junit.test.integration.runner.order;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.eclipse.reddeer.junit.requirement.Requirement;
+import org.eclipse.reddeer.junit.test.integration.runner.order.OrderRunnerRequirement.RequirementOrderAnnotation;
+
+public class OrderRunnerRequirement implements Requirement<RequirementOrderAnnotation> {
+
+	@Retention(RetentionPolicy.RUNTIME)
+	@Target(ElementType.TYPE)
+	public @interface RequirementOrderAnnotation {
+
+	}
+
+	@Override
+	public void fulfill() {
+		TestSequence.addFulfill(OrderRunnerRequirement.class);
+	}
+
+	@Override
+	public void setDeclaration(RequirementOrderAnnotation declaration) {
+	}
+
+	@Override
+	public void cleanUp() {
+		TestSequence.addCleanup(OrderRunnerRequirement.class);
+	}
+
+	@Override
+	public void runBeforeClass() {
+		TestSequence.addReqBeforeClass(OrderRunnerRequirement.class);
+	}
+
+	@Override
+	public void runBefore() {
+		TestSequence.addReqBefore(OrderRunnerRequirement.class);
+	}
+
+	@Override
+	public void runAfterClass() {
+		TestSequence.addReqAfterClass(OrderRunnerRequirement.class);
+	}
+
+	@Override
+	public void runAfter() {
+		TestSequence.addReqAfter(OrderRunnerRequirement.class);
+	}
+
+	@Override
+	public RequirementOrderAnnotation getDeclaration() {
+		return null;
+	}
+
+}

tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/RequirementsOrderRunnerSuite.java

@@ -0,0 +1,143 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Red Hat, Inc and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Red Hat Inc. - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.reddeer.junit.test.integration.runner.order;
+
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createAfter;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createAfterClass;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createBefore;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createBeforeClass;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createCleanup;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createFulfill;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createIAfter;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createIAfterClass;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createIBefore;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createIBeforeClass;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createReqAfter;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createReqAfterClass;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createReqBefore;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createReqBeforeClass;
+import static org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence.createTest;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.eclipse.reddeer.common.exception.RedDeerException;
+import org.eclipse.reddeer.junit.internal.configuration.SuiteConfiguration;
+import org.eclipse.reddeer.junit.test.integration.runner.IAfterTestImpl;
+import org.eclipse.reddeer.junit.test.integration.runner.IBeforeTestImpl;
+import org.eclipse.reddeer.junit.test.integration.runner.order.OrderRunnerRequirement.RequirementOrderAnnotation;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.RunnerBuilder;
+
+public class RequirementsOrderRunnerSuite extends TestSequenceRedDeerSuite {
+
+	private static final List<Object> expectedSequence;
+
+	static {
+		expectedSequence = new ArrayList<Object>();
+		expectedSequence.add(createIBeforeClass(IBeforeTestImpl.class));
+		expectedSequence.add(createFulfill(OrderRunnerRequirement.class));
+		expectedSequence.add(createReqBeforeClass(OrderRunnerRequirement.class));
+		expectedSequence.add(createBeforeClass(RequirementsOrderRunnerSuite.class));
+
+		expectedSequence.add(createIBefore(IBeforeTestImpl.class));
+		expectedSequence.add(createReqBefore(OrderRunnerRequirement.class));
+		expectedSequence.add(createBefore(RequirementsOrderRunnerSuite.class));
+		expectedSequence.add(createTest(RequirementsOrderRunnerSuite.class));
+		expectedSequence.add(createAfter(RequirementsOrderRunnerSuite.class));
+		expectedSequence.add(createReqAfter(OrderRunnerRequirement.class));
+		expectedSequence.add(createIAfter(IAfterTestImpl.class));
+
+		expectedSequence.add(createAfterClass(RequirementsOrderRunnerSuite.class));
+		expectedSequence.add(createReqAfterClass(OrderRunnerRequirement.class));
+		expectedSequence.add(createCleanup(OrderRunnerRequirement.class));
+		expectedSequence.add(createIAfterClass(IAfterTestImpl.class));
+	}
+
+	@Override
+	protected List<?> getExpectedSequence() {
+		return expectedSequence;
+	}
+
+	public RequirementsOrderRunnerSuite(Class<?> clazz, RunnerBuilder builder,
+			SuiteConfiguration config) throws InitializationError {
+		super(clazz, builder, config);
+	}
+
+	public RequirementsOrderRunnerSuite(Class<?> clazz, RunnerBuilder builder)
+			throws InitializationError {
+		super(clazz, builder);
+	}
+
+	public static List<?> getStaticExpectedSequence() {
+		return expectedSequence;
+	}
+
+	@RunWith(RequirementsOrderRunnerSuite.class)
+	@RequirementOrderAnnotation
+	public static class RequirementOrderParentTest {
+
+		@BeforeClass
+		public static void beforeClass(){
+			TestSequence.addBeforeClass(RequirementsOrderRunnerSuite.class);
+		}
+
+		@Before
+		public void before(){
+			TestSequence.addBefore(RequirementsOrderRunnerSuite.class);
+		}
+
+		@After
+		public void after(){
+			TestSequence.addAfter(RequirementsOrderRunnerSuite.class);
+		}
+
+		@AfterClass
+		public static void afterClass(){
+			TestSequence.addAfterClass(RequirementsOrderRunnerSuite.class);
+		}
+	}
+
+	public static class RequirementOrderPassedTest extends RequirementOrderParentTest {
+
+		@Test
+		public void test(){
+			TestSequence.addTest(RequirementsOrderRunnerSuite.class);
+		}
+	}
+
+	public static class RequirementOrderFailedTest extends RequirementOrderParentTest {
+
+		@Test
+		public void test(){
+			TestSequence.addTest(RequirementsOrderRunnerSuite.class);
+			assertTrue("Test in failure", false);
+		}
+	}
+
+	public static class RequirementOrderErroneousTest extends RequirementOrderParentTest {
+
+		@Test
+		public void test(){
+			TestSequence.addTest(RequirementsOrderRunnerSuite.class);
+			throw new RedDeerException("Test in error");
+		}
+	}
+
+
+}

tests/org.eclipse.reddeer.junit.test/src/org/eclipse/reddeer/junit/test/integration/runner/order/testcase/RequirementOrderCallWithFailsTest.java

@@ -0,0 +1,86 @@
+/*******************************************************************************
+ * Copyright (c) 2018 Red Hat, Inc and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Red Hat Inc. - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.reddeer.junit.test.integration.runner.order.testcase;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.eclipse.reddeer.junit.internal.configuration.SuiteConfiguration;
+import org.eclipse.reddeer.junit.runner.RedDeerSuite;
+import org.eclipse.reddeer.junit.test.integration.runner.order.RequirementsOrderRunnerSuite;
+import org.eclipse.reddeer.junit.test.integration.runner.order.RequirementsOrderRunnerSuite.RequirementOrderErroneousTest;
+import org.eclipse.reddeer.junit.test.integration.runner.order.RequirementsOrderRunnerSuite.RequirementOrderFailedTest;
+import org.eclipse.reddeer.junit.test.integration.runner.order.RequirementsOrderRunnerSuite.RequirementOrderPassedTest;
+import org.eclipse.reddeer.junit.test.integration.runner.order.TestSequence;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runner.Runner;
+import org.junit.runner.notification.RunNotifier;
+import org.junit.runners.model.InitializationError;
+
+/**
+ * Tests order of calling befores and afters of test class 
+ * combined with requirement calling of those methods during passed/failed/erroneous tests.
+ * @author odockal
+ *
+ */
+@RunWith(RedDeerSuite.class)
+public class RequirementOrderCallWithFailsTest {
+
+	@Before
+	public void setupSequence() {
+		TestSequence.getRealSequence().clear();
+	}
+
+	@Test
+	public void testPassingTest() throws InitializationError {
+		SuiteConfiguration config = new SuiteConfiguration(RequirementOrderPassedTest.class);
+		List<Runner> runners = RedDeerSuite.createSuites(RequirementOrderPassedTest.class, config);
+
+		Runner myRunner = runners.get(0);
+		RunNotifier notifier = new RunNotifier();
+		myRunner.run(notifier); // runner.run executes the test and static values could be retrieved
+		String sequenceDiff = TestSequence.diffRealSequence(RequirementsOrderRunnerSuite.getStaticExpectedSequence());
+		assertTrue("Test sequence is different than expected. " + sequenceDiff , sequenceDiff == null);
+	}
+
+	@Test
+	public void testFailingTest() throws InitializationError {
+		SuiteConfiguration config = new SuiteConfiguration(RequirementOrderFailedTest.class);
+		List<Runner> runners = RedDeerSuite.createSuites(RequirementOrderFailedTest.class, config);
+
+		Runner myRunner = runners.get(0);
+		RunNotifier notifier = new RunNotifier();
+		myRunner.run(notifier); // runner.run executes the test and static values could be retrieved
+		String sequenceDiff = TestSequence.diffRealSequence(RequirementsOrderRunnerSuite.getStaticExpectedSequence());
+		assertTrue("Test sequence is different than expected. " + sequenceDiff , sequenceDiff == null);
+	}
+
+	@Test
+	public void testErroneousTest() throws InitializationError {
+		SuiteConfiguration config = new SuiteConfiguration(RequirementOrderErroneousTest.class);
+		List<Runner> runners = RedDeerSuite.createSuites(RequirementOrderErroneousTest.class, config);
+
+		Runner myRunner = runners.get(0);
+		RunNotifier notifier = new RunNotifier();
+		myRunner.run(notifier); // runner.run executes the test and static values could be retrieved
+		String sequenceDiff = TestSequence.diffRealSequence(RequirementsOrderRunnerSuite.getStaticExpectedSequence());
+		assertTrue("Test sequence is different than expected. " + sequenceDiff , sequenceDiff == null);
+	}
+
+	@After
+	public void clearSequence() {
+		TestSequence.getRealSequence().clear();
+	}
+}