From 817214c13c3262f00635b2b474e2f443ea2220f0 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 16 Apr 2020 11:49:00 -0500 Subject: [PATCH 1/4] MQE-2086: Reduce amount of attachments in Allure reports - Implemented new VERBOSE_ARTIFACTS FLAG --- dev/tests/functional/standalone_bootstrap.php | 3 +++ .../Allure/Adapter/MagentoAllureAdapter.php | 10 ++++++++++ src/Magento/FunctionalTestingFramework/_bootstrap.php | 3 +++ 3 files changed, 16 insertions(+) diff --git a/dev/tests/functional/standalone_bootstrap.php b/dev/tests/functional/standalone_bootstrap.php index 34516236e..58030231d 100755 --- a/dev/tests/functional/standalone_bootstrap.php +++ b/dev/tests/functional/standalone_bootstrap.php @@ -53,6 +53,9 @@ defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 30); $env->setEnvironmentVariable('WAIT_TIMEOUT', WAIT_TIMEOUT); + defined('VERBOSE_ARTIFACTS') || define('VERBOSE_ARTIFACTS', false); + $env->setEnvironmentVariable('VERBOSE_ARTIFACTS', VERBOSE_ARTIFACTS); + try { new DateTimeZone(DEFAULT_TIMEZONE); } catch (\Exception $e) { diff --git a/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php b/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php index e9e1c344c..5b5a96b22 100644 --- a/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php +++ b/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php @@ -10,6 +10,7 @@ use Magento\FunctionalTestingFramework\Test\Objects\ActionGroupObject; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; use Magento\FunctionalTestingFramework\Util\TestGenerator; +use Yandex\Allure\Adapter\Model\Provider; use Yandex\Allure\Adapter\Model\Status; use Yandex\Allure\Adapter\Model\Step; use Yandex\Allure\Codeception\AllureCodeception; @@ -250,6 +251,8 @@ public function testError(FailEvent $failEvent) */ public function testEnd() { + // Peek top of testCaseStorage to check of failure + $testFailed = $this->getLifecycle()->getTestCaseStorage()->get()->getFailure(); // Pops top of stepStorage, need to add it back in after processing $rootStep = $this->getLifecycle()->getStepStorage()->pollLast(); $formattedSteps = []; @@ -257,6 +260,13 @@ public function testEnd() $actionGroupStepKey = null; foreach ($rootStep->getSteps() as $step) { + //Remove Attachments if verbose flag is not true AND test did not fail + if (getenv('VERBOSE_ARTIFACTS') !== true && $testFailed === null) { + foreach ($step->getAttachments() as $index => $attachment) { + $step->removeAttachment($index); + unlink(Provider::getOutputDirectory() . DIRECTORY_SEPARATOR . $attachment->getSource()); + } + } $stepKey = str_replace($actionGroupStepKey, '', $step->getName()); if ($stepKey !== '[]' && $stepKey !== null) { $step->setName($stepKey); diff --git a/src/Magento/FunctionalTestingFramework/_bootstrap.php b/src/Magento/FunctionalTestingFramework/_bootstrap.php index e35b20bad..b655fb28e 100644 --- a/src/Magento/FunctionalTestingFramework/_bootstrap.php +++ b/src/Magento/FunctionalTestingFramework/_bootstrap.php @@ -53,6 +53,9 @@ defined('WAIT_TIMEOUT') || define('WAIT_TIMEOUT', 30); $env->setEnvironmentVariable('WAIT_TIMEOUT', WAIT_TIMEOUT); + defined('VERBOSE_ARTIFACTS') || define('VERBOSE_ARTIFACTS', false); + $env->setEnvironmentVariable('VERBOSE_ARTIFACTS', VERBOSE_ARTIFACTS); + try { new DateTimeZone(DEFAULT_TIMEZONE); } catch (\Exception $e) { From b84ced35a73ff9d915bcd1c9313d6822e9b12a63 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Thu, 16 Apr 2020 12:40:57 -0500 Subject: [PATCH 2/4] MQE-2086: Reduce amount of attachments in Allure reports - Static check fixes --- .../Allure/Adapter/MagentoAllureAdapter.php | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php b/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php index 5b5a96b22..bc0d0a91c 100644 --- a/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php +++ b/src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php @@ -10,6 +10,7 @@ use Magento\FunctionalTestingFramework\Test\Objects\ActionGroupObject; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; use Magento\FunctionalTestingFramework\Util\TestGenerator; +use Yandex\Allure\Adapter\Model\Failure; use Yandex\Allure\Adapter\Model\Provider; use Yandex\Allure\Adapter\Model\Status; use Yandex\Allure\Adapter\Model\Step; @@ -260,13 +261,7 @@ public function testEnd() $actionGroupStepKey = null; foreach ($rootStep->getSteps() as $step) { - //Remove Attachments if verbose flag is not true AND test did not fail - if (getenv('VERBOSE_ARTIFACTS') !== true && $testFailed === null) { - foreach ($step->getAttachments() as $index => $attachment) { - $step->removeAttachment($index); - unlink(Provider::getOutputDirectory() . DIRECTORY_SEPARATOR . $attachment->getSource()); - } - } + $this->removeAttachments($step, $testFailed); $stepKey = str_replace($actionGroupStepKey, '', $step->getName()); if ($stepKey !== '[]' && $stepKey !== null) { $step->setName($stepKey); @@ -364,4 +359,21 @@ private function retrieveStepKey($stepLine) return $stepKey; } + + /** + * Removes attachments from step depending on MFTF configuration + * @param Step $step + * @param Failure $testFailed + * @return void + */ + private function removeAttachments($step, $testFailed) + { + //Remove Attachments if verbose flag is not true AND test did not fail + if (getenv('VERBOSE_ARTIFACTS') !== true && $testFailed === null) { + foreach ($step->getAttachments() as $index => $attachment) { + $step->removeAttachment($index); + unlink(Provider::getOutputDirectory() . DIRECTORY_SEPARATOR . $attachment->getSource()); + } + } + } } From 0fca63b1d22918b0198ca7a903eeb50cba758c72 Mon Sep 17 00:00:00 2001 From: Kevin Kozan Date: Fri, 17 Apr 2020 09:16:10 -0500 Subject: [PATCH 3/4] MQE-2086: Reduce amount of attachments in Allure reports - Docs update --- docs/configuration.md | 14 ++++++++++++++ etc/config/.env.example | 3 +++ 2 files changed, 17 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 9466f2bcc..11882084e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -299,6 +299,20 @@ Example: CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default ``` +### VERBOSE_ARTIFACTS + +Determines if passed tests should still have all their Allure artifacts. These artifacts include `.txt` attachments for things like `dontSee` actions and `createData` actions. + +If enabled, all tests will have all of their normal Allure artifacts. + +If disabled, passed tests will have their Allure artifacts trimmed. Failed tests will still contain all their artifacts. + +This is set `false` by default. + +```conf +VERBOSE_ARTIFACTS=true +``` + ### ENABLE_BROWSER_LOG Enables addition of browser logs to Allure steps diff --git a/etc/config/.env.example b/etc/config/.env.example index 349d7da9c..86df31b3d 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -59,6 +59,9 @@ MODULE_WHITELIST=Magento_Framework,ConfigurableProductWishlist,ConfigurableProdu #*** Default timeout for wait actions #WAIT_TIMEOUT=30 +#*** Uncomment and set to enable all tests, regardless of passing status, to have all their Allure artifacts. +#VERBOSE_ARTIFACTS=true + #*** Uncomment and set to enable browser log entries on actions in Allure. Blacklist is used to filter logs of a specific "source" #ENABLE_BROWSER_LOG=true #BROWSER_LOG_BLACKLIST=other From 4b8581fc74ecb326c7b256bda91fea49b052dac5 Mon Sep 17 00:00:00 2001 From: Donald Booth Date: Mon, 20 Apr 2020 13:28:20 -0500 Subject: [PATCH 4/4] grammar --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 11882084e..d2ce5e1a5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -301,7 +301,7 @@ CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default ### VERBOSE_ARTIFACTS -Determines if passed tests should still have all their Allure artifacts. These artifacts include `.txt` attachments for things like `dontSee` actions and `createData` actions. +Determines if passed tests should still have all their Allure artifacts. These artifacts include `.txt` attachments for `dontSee` actions and `createData` actions. If enabled, all tests will have all of their normal Allure artifacts.