MQE-830: Output error details about test failure

KevinBKozan · web-flow · commit 6ab66f33532b · 2018-03-28T17:23:26.000+03:00
- Added new extension to allow for pulling and operation on testContext at test runtime
- Tests now no longer allow for exceptions in _after block to overwrite the original exception (case when test fails, runs _after and fails there too)
- Tests no longer run the _after block if the test failure was in the _after block to begin
diff --git a/dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt b/dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt
@@ -57,7 +57,6 @@ class ActionGroupWithDataOverrideTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/ActionGroupWithDataTest.txt b/dev/tests/verification/Resources/ActionGroupWithDataTest.txt
@@ -57,7 +57,6 @@ class ActionGroupWithDataTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt b/dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt
@@ -57,7 +57,6 @@ class ActionGroupWithNoDefaultTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/ActionGroupWithPersistedData.txt b/dev/tests/verification/Resources/ActionGroupWithPersistedData.txt
@@ -57,7 +57,6 @@ class ActionGroupWithPersistedDataCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt b/dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt
@@ -57,7 +57,6 @@ class ActionGroupWithTopLevelPersistedDataCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt b/dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt
@@ -57,7 +57,6 @@ class ArgumentWithSameNameAsElementCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/BasicFunctionalTest.txt b/dev/tests/verification/Resources/BasicFunctionalTest.txt
@@ -46,7 +46,6 @@ class BasicFunctionalTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/BasicMergeTest.txt b/dev/tests/verification/Resources/BasicMergeTest.txt
@@ -48,7 +48,6 @@ class BasicMergeTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/HookActionsTest.txt b/dev/tests/verification/Resources/HookActionsTest.txt
@@ -73,7 +73,6 @@ class HookActionsTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/MergedActionGroupTest.txt b/dev/tests/verification/Resources/MergedActionGroupTest.txt
@@ -57,7 +57,6 @@ class MergedActionGroupTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/MergedReferencesTest.txt b/dev/tests/verification/Resources/MergedReferencesTest.txt
@@ -46,7 +46,6 @@ class MergedReferencesTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/dev/tests/verification/Resources/MultipleActionGroupsTest.txt b/dev/tests/verification/Resources/MultipleActionGroupsTest.txt
@@ -57,7 +57,6 @@ class MultipleActionGroupsTestCest
 	public function _failed(AcceptanceTester $I)
 	{
 		$I->saveScreenshot();
-		$this->_after($I);
 	}
 
 	/**
diff --git a/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php b/src/Magento/FunctionalTestingFramework/Extension/TestContextExtension.php
@@ -0,0 +1,71 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace Magento\FunctionalTestingFramework\Extension;
+
+use \Codeception\Events;
+
+/**
+ * Class TestContextExtension
+ * @SuppressWarnings(PHPMD.UnusedPrivateField)
+ */
+class TestContextExtension extends \Codeception\Extension
+{
+    const TEST_PHASE_AFTER = "_after";
+
+    /**
+     * Codeception Events Mapping to methods
+     * @var array
+     */
+    public static $events = [
+        Events::TEST_FAIL => 'testFail',
+    ];
+
+    /**
+     * Codeception event listener function, triggered on test failure.
+     * @param \Codeception\Event\FailEvent $e
+     * @return void
+     */
+    public function testFail(\Codeception\Event\FailEvent $e)
+    {
+        $cest = $e->getTest();
+        $context = $this->extractContext($e->getFail()->getTrace(), $cest->getTestMethod());
+        // Do not attempt to run _after if failure was in the _after block
+        // Try to run _after but catch exceptions to prevent them from overwriting original failure.
+        if ($context != TestContextExtension::TEST_PHASE_AFTER) {
+            try {
+                $actorClass = $e->getTest()->getMetadata()->getCurrent('actor');
+                $I = new $actorClass($cest->getScenario());
+                call_user_func(\Closure::bind(
+                    function () use ($cest, $I) {
+                        $cest->executeHook($I, 'after');
+                    },
+                    null,
+                    $cest
+                ));
+            } catch (\Exception $e) {
+                // Do not rethrow Exception
+            }
+        }
+    }
+
+    /**
+     * Extracts hook method from trace, looking specifically for the cest class given.
+     * @param array $trace
+     * @param string $class
+     * @return string
+     */
+    public function extractContext($trace, $class)
+    {
+        foreach ($trace as $entry) {
+            $traceClass = $entry["class"] ?? null;
+            if (strpos($traceClass, $class) != 0) {
+                return $entry["function"];
+            }
+        }
+        return null;
+    }
+}
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -1373,6 +1373,7 @@ private function stripAndSplitReference($reference, $delimiter)
      * @return string
      * @throws TestReferenceException
      * @throws \Exception
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
     private function generateHooksPhp($hookObjects)
     {
@@ -1414,12 +1415,6 @@ private function generateHooksPhp($hookObjects)
                 throw new TestReferenceException($e->getMessage() . " in Element \"" . $type . "\"");
             }
 
-            if ($hookObject->getType() == TestObjectExtractor::TEST_FAILED_HOOK) {
-                $steps.="\t\t";
-                $steps.='$this->_after($I);';
-                $steps.="\n";
-            }
-
             $hooks .= sprintf("\tpublic function _{$type}(%s)\n", $dependencies);
             $hooks .= "\t{\n";
             $hooks .= $steps;

Original file line number	Diff line number	Diff line change
`@@ -57,7 +57,6 @@ class ActionGroupWithDataOverrideTestCest`
`57`	`57`	`public function _failed(AcceptanceTester $I)`
`58`	`58`	`{`
`59`	`59`	`$I->saveScreenshot();`
`60`		`- $this->_after($I);`
`61`	`60`	`}`
`62`	`61`
`63`	`62`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -57,7 +57,6 @@ class ArgumentWithSameNameAsElementCest`
`57`	`57`	`public function _failed(AcceptanceTester $I)`
`58`	`58`	`{`
`59`	`59`	`$I->saveScreenshot();`
`60`		`- $this->_after($I);`
`61`	`60`	`}`
`62`	`61`
`63`	`62`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,6 @@ class BasicFunctionalTestCest`
`46`	`46`	`public function _failed(AcceptanceTester $I)`
`47`	`47`	`{`
`48`	`48`	`$I->saveScreenshot();`
`49`		`- $this->_after($I);`
`50`	`49`	`}`
`51`	`50`
`52`	`51`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,6 @@ class BasicMergeTestCest`
`48`	`48`	`public function _failed(AcceptanceTester $I)`
`49`	`49`	`{`
`50`	`50`	`$I->saveScreenshot();`
`51`		`- $this->_after($I);`
`52`	`51`	`}`
`53`	`52`
`54`	`53`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -73,7 +73,6 @@ class HookActionsTestCest`
`73`	`73`	`public function _failed(AcceptanceTester $I)`
`74`	`74`	`{`
`75`	`75`	`$I->saveScreenshot();`
`76`		`- $this->_after($I);`
`77`	`76`	`}`
`78`	`77`
`79`	`78`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -57,7 +57,6 @@ class MergedActionGroupTestCest`
`57`	`57`	`public function _failed(AcceptanceTester $I)`
`58`	`58`	`{`
`59`	`59`	`$I->saveScreenshot();`
`60`		`- $this->_after($I);`
`61`	`60`	`}`
`62`	`61`
`63`	`62`	`/**`