diff --git a/.gitignore b/.gitignore index c1adae14d..d39929c6f 100755 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ coverage/ codeception.yml dev/tests/functional/MFTF.suite.yml dev/tests/functional/_output +dev/mftf.log dev/tests/mftf.log \ No newline at end of file diff --git a/dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt b/dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt index 0c4ff37af..c939f421e 100644 --- a/dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt +++ b/dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class ActionGroupWithDataOverrideTestCest diff --git a/dev/tests/verification/Resources/ActionGroupWithDataTest.txt b/dev/tests/verification/Resources/ActionGroupWithDataTest.txt index 5f7d75899..d11eff9a8 100644 --- a/dev/tests/verification/Resources/ActionGroupWithDataTest.txt +++ b/dev/tests/verification/Resources/ActionGroupWithDataTest.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class ActionGroupWithDataTestCest diff --git a/dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt b/dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt index 82fde4afd..8739d8fa0 100644 --- a/dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt +++ b/dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class ActionGroupWithNoDefaultTestCest diff --git a/dev/tests/verification/Resources/ActionGroupWithPersistedData.txt b/dev/tests/verification/Resources/ActionGroupWithPersistedData.txt index fcd4f74f9..63fd6af52 100644 --- a/dev/tests/verification/Resources/ActionGroupWithPersistedData.txt +++ b/dev/tests/verification/Resources/ActionGroupWithPersistedData.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class ActionGroupWithPersistedDataCest diff --git a/dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt b/dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt index 6588d490b..1881a392a 100644 --- a/dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt +++ b/dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class ActionGroupWithTopLevelPersistedDataCest diff --git a/dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt b/dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt index bc2890967..90b195688 100644 --- a/dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt +++ b/dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class ArgumentWithSameNameAsElementCest diff --git a/dev/tests/verification/Resources/BasicActionGroupTest.txt b/dev/tests/verification/Resources/BasicActionGroupTest.txt index 45ccc72a6..259016681 100644 --- a/dev/tests/verification/Resources/BasicActionGroupTest.txt +++ b/dev/tests/verification/Resources/BasicActionGroupTest.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class BasicActionGroupTestCest diff --git a/dev/tests/verification/Resources/BasicMergeTest.txt b/dev/tests/verification/Resources/BasicMergeTest.txt index ae3a69096..ad5a0c189 100644 --- a/dev/tests/verification/Resources/BasicMergeTest.txt +++ b/dev/tests/verification/Resources/BasicMergeTest.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") + * @Title("BasicMergeTest") * @group functional * @group mergeTest */ diff --git a/dev/tests/verification/Resources/ChildExtendedTestAddHooks.txt b/dev/tests/verification/Resources/ChildExtendedTestAddHooks.txt index 3b7015529..bf3d3a8c2 100644 --- a/dev/tests/verification/Resources/ChildExtendedTestAddHooks.txt +++ b/dev/tests/verification/Resources/ChildExtendedTestAddHooks.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("Parent") + * @Title("ChildExtendedTestAddHooks") * @group Parent */ class ChildExtendedTestAddHooksCest diff --git a/dev/tests/verification/Resources/ChildExtendedTestMerging.txt b/dev/tests/verification/Resources/ChildExtendedTestMerging.txt index 90943edfc..1efe59cb5 100644 --- a/dev/tests/verification/Resources/ChildExtendedTestMerging.txt +++ b/dev/tests/verification/Resources/ChildExtendedTestMerging.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("Child") + * @Title("ChildExtendedTestMerging") * @group Child */ class ChildExtendedTestMergingCest diff --git a/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt b/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt index 007db6bf5..3bc7b8a0a 100644 --- a/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt +++ b/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("Child") + * @Title("ChildExtendedTestNoParent") * @group Child * @group skip */ diff --git a/dev/tests/verification/Resources/ChildExtendedTestRemoveAction.txt b/dev/tests/verification/Resources/ChildExtendedTestRemoveAction.txt index a9ef163b4..f24c3e6d1 100644 --- a/dev/tests/verification/Resources/ChildExtendedTestRemoveAction.txt +++ b/dev/tests/verification/Resources/ChildExtendedTestRemoveAction.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("Child") + * @Title("ChildExtendedTestRemoveAction") * @group Child */ class ChildExtendedTestRemoveActionCest diff --git a/dev/tests/verification/Resources/ChildExtendedTestRemoveHookAction.txt b/dev/tests/verification/Resources/ChildExtendedTestRemoveHookAction.txt index 4a778a9fc..9d207e13f 100644 --- a/dev/tests/verification/Resources/ChildExtendedTestRemoveHookAction.txt +++ b/dev/tests/verification/Resources/ChildExtendedTestRemoveHookAction.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("Child") + * @Title("ChildExtendedTestRemoveHookAction") * @group Child */ class ChildExtendedTestRemoveHookActionCest diff --git a/dev/tests/verification/Resources/ChildExtendedTestReplace.txt b/dev/tests/verification/Resources/ChildExtendedTestReplace.txt index b41a22323..6f3c3c9b3 100644 --- a/dev/tests/verification/Resources/ChildExtendedTestReplace.txt +++ b/dev/tests/verification/Resources/ChildExtendedTestReplace.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("Child") + * @Title("ChildExtendedTestReplace") * @group Child */ class ChildExtendedTestReplaceCest diff --git a/dev/tests/verification/Resources/ChildExtendedTestReplaceHook.txt b/dev/tests/verification/Resources/ChildExtendedTestReplaceHook.txt index 384aa8d5f..2a5937381 100644 --- a/dev/tests/verification/Resources/ChildExtendedTestReplaceHook.txt +++ b/dev/tests/verification/Resources/ChildExtendedTestReplaceHook.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("Child") + * @Title("ChildExtendedTestReplaceHook") * @group Child */ class ChildExtendedTestReplaceHookCest diff --git a/dev/tests/verification/Resources/MergedActionGroupTest.txt b/dev/tests/verification/Resources/MergedActionGroupTest.txt index f6c8f2351..f1d73dba5 100644 --- a/dev/tests/verification/Resources/MergedActionGroupTest.txt +++ b/dev/tests/verification/Resources/MergedActionGroupTest.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class MergedActionGroupTestCest diff --git a/dev/tests/verification/Resources/MergedReferencesTest.txt b/dev/tests/verification/Resources/MergedReferencesTest.txt index 530a23b2e..fe0c9498f 100644 --- a/dev/tests/verification/Resources/MergedReferencesTest.txt +++ b/dev/tests/verification/Resources/MergedReferencesTest.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") + * @Title("MergedReferencesTest") * @group functional */ class MergedReferencesTestCest diff --git a/dev/tests/verification/Resources/MultipleActionGroupsTest.txt b/dev/tests/verification/Resources/MultipleActionGroupsTest.txt index 8bb180f5d..18f6df3a9 100644 --- a/dev/tests/verification/Resources/MultipleActionGroupsTest.txt +++ b/dev/tests/verification/Resources/MultipleActionGroupsTest.txt @@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("A Functional Cest") * @group functional */ class MultipleActionGroupsTestCest diff --git a/dev/tests/verification/Resources/ParentExtendedTest.txt b/dev/tests/verification/Resources/ParentExtendedTest.txt index 70ef6ada8..a79783217 100644 --- a/dev/tests/verification/Resources/ParentExtendedTest.txt +++ b/dev/tests/verification/Resources/ParentExtendedTest.txt @@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel; use Yandex\Allure\Adapter\Annotation\TestCaseId; /** - * @Title("Parent") + * @Title("ParentExtendedTest") * @group Parent */ class ParentExtendedTestCest diff --git a/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest.xml b/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest.xml index 63f800983..8c24794e1 100644 --- a/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest.xml +++ b/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest.xml @@ -11,7 +11,6 @@ - <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> @@ -27,7 +26,6 @@ <test name="ActionGroupWithDataTest"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> @@ -46,7 +44,6 @@ <test name="ActionGroupWithDataOverrideTest"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> @@ -67,7 +64,6 @@ <test name="ActionGroupWithNoDefaultTest"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> @@ -88,7 +84,6 @@ <test name="ActionGroupWithPersistedData"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> @@ -108,7 +103,6 @@ <test name="ActionGroupWithTopLevelPersistedData"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> @@ -127,7 +121,6 @@ <test name="MultipleActionGroupsTest"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> @@ -149,7 +142,6 @@ <test name="MergedActionGroupTest"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> @@ -168,7 +160,6 @@ <test name="ArgumentWithSameNameAsElement"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> <group value="functional"/> <features value="Action Group Functional Cest"/> <stories value="MQE-433"/> diff --git a/dev/tests/verification/TestModule/Test/ExtendedFunctionalTest.xml b/dev/tests/verification/TestModule/Test/ExtendedFunctionalTest.xml index dcf929188..602b9a02c 100644 --- a/dev/tests/verification/TestModule/Test/ExtendedFunctionalTest.xml +++ b/dev/tests/verification/TestModule/Test/ExtendedFunctionalTest.xml @@ -11,7 +11,7 @@ <test name="ParentExtendedTest"> <annotations> <severity value="AVERAGE"/> - <title value="Parent"/> + <title value="ParentExtendedTest"/> <group value="Parent"/> <features value="Parent"/> <stories value="Parent"/> @@ -28,7 +28,7 @@ <test name="ChildExtendedTestReplace" extends="ParentExtendedTest"> <annotations> <severity value="MINOR"/> - <title value="Child"/> + <title value="ChildExtendedTestReplace"/> <group value="Child"/> <features value="Child"/> <stories value="Child"/> @@ -39,7 +39,7 @@ <test name="ChildExtendedTestReplaceHook" extends="ParentExtendedTest"> <annotations> <severity value="MINOR"/> - <title value="Child"/> + <title value="ChildExtendedTestReplaceHook"/> <group value="Child"/> <features value="Child"/> <stories value="Child"/> @@ -52,7 +52,7 @@ <test name="ChildExtendedTestMerging" extends="ParentExtendedTest"> <annotations> <severity value="MINOR"/> - <title value="Child"/> + <title value="ChildExtendedTestMerging"/> <group value="Child"/> <features value="Child"/> <stories value="Child"/> @@ -69,7 +69,7 @@ <test name="ChildExtendedTestRemoveAction" extends="ParentExtendedTest"> <annotations> <severity value="CRITICAL"/> - <title value="Child"/> + <title value="ChildExtendedTestRemoveAction"/> <group value="Child"/> <features value="Child"/> <stories value="Child"/> @@ -80,7 +80,7 @@ <test name="ParentExtendedTestNoHooks"> <annotations> <severity value="AVERAGE"/> - <title value="Parent"/> + <title value="ParentExtendedTestNoHooks"/> <group value="Parent"/> <features value="Parent"/> <stories value="Parent"/> @@ -91,7 +91,7 @@ <test name="ChildExtendedTestAddHooks"> <annotations> <severity value="AVERAGE"/> - <title value="Parent"/> + <title value="ChildExtendedTestAddHooks"/> <group value="Parent"/> <features value="Parent"/> <stories value="Parent"/> @@ -107,7 +107,7 @@ <test name="ChildExtendedTestRemoveHookAction" extends="ParentExtendedTest"> <annotations> <severity value="CRITICAL"/> - <title value="Child"/> + <title value="ChildExtendedTestRemoveHookAction"/> <group value="Child"/> <features value="Child"/> <stories value="Child"/> @@ -120,7 +120,7 @@ <test name="ChildExtendedTestNoParent" extends="ThisTestDoesNotExist"> <annotations> <severity value="CRITICAL"/> - <title value="Child"/> + <title value="ChildExtendedTestNoParent"/> <group value="Child"/> <features value="Child"/> <stories value="Child"/> diff --git a/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml b/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml index 553f3d251..f7997946c 100644 --- a/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml +++ b/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml @@ -11,7 +11,7 @@ <test name="BasicMergeTest"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> + <title value="BasicMergeTest"/> <group value="functional"/> <features value="Merge Functional Cest"/> <stories value="MQE-433"/> @@ -31,7 +31,7 @@ <test name="MergedReferencesTest"> <annotations> <severity value="CRITICAL"/> - <title value="A Functional Cest"/> + <title value="MergedReferencesTest"/> <group value="functional"/> <features value="Merge Functional Cest"/> <stories value="MQE-433"/> diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php index 025ebb079..e7ccc90ef 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php +++ b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php @@ -13,6 +13,7 @@ use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser; use Magento\FunctionalTestingFramework\Test\Util\ObjectExtensionUtil; use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor; +use Magento\FunctionalTestingFramework\Test\Util\AnnotationExtractor; /** * Class TestObjectHandler @@ -142,6 +143,7 @@ private function initTestData() $this->tests[$testName] = $testObjectExtractor->extractTestData($testData); } + $testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness(); } /** diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php index 6e98c78f3..1718bf059 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php @@ -6,11 +6,20 @@ namespace Magento\FunctionalTestingFramework\Test\Util; +use Magento\FunctionalTestingFramework\Exceptions\XmlException; + /** * Class AnnotationExtractor */ class AnnotationExtractor extends BaseObjectExtractor { + /** + * Mappings of all Test => title mappings, indexed by Story + * e.g. $storyToTitleMappings['storyAnnotation'] = ['testName' => 'titleAnnotation'] + * @var array + */ + private $storyToTitleMappings = []; + const ANNOTATION_VALUE = 'value'; const MAGENTO_TO_ALLURE_SEVERITY_MAP = [ "BLOCKER" => "BLOCKER", @@ -33,11 +42,11 @@ public function __construct() * can be found in both Tests and their child element tests. * * @param array $testAnnotations + * @param string $filename * @return array */ - public function extractAnnotations($testAnnotations) + public function extractAnnotations($testAnnotations, $filename) { - $annotationObjects = []; $annotations = $this->stripDescriptorTags($testAnnotations, self::NODE_NAME); @@ -58,10 +67,52 @@ public function extractAnnotations($testAnnotations) } $annotationObjects[$annotationKey] = $annotationValues; } - + $this->addStoryTitleToMap($annotationObjects, $filename); return $annotationObjects; } + /** + * Adds story/title/filename combination to static map + * @param array $annotations + * @param string $filename + * @return void + */ + public function addStoryTitleToMap($annotations, $filename) + { + if (isset($annotations['stories']) && isset($annotations['title'])) { + $story = $annotations['stories'][0]; + $title = $annotations['title'][0]; + $this->storyToTitleMappings[$story . "/" . $title][] = $filename; + } + } + + /** + * Validates that all Story/Title combinations are unique, builds list of violators if found. + * @throws XmlException + * @return void + */ + public function validateStoryTitleUniqueness() + { + $dupes = []; + + foreach ($this->storyToTitleMappings as $storyTitle => $files) { + if (count($files) > 1) { + $dupes[$storyTitle] = "'" . implode("', '", $files) . "'"; + } + } + if (!empty($dupes)) { + $message = "Story and Title annotation pairs must be unique:\n\n"; + foreach ($dupes as $storyTitle => $tests) { + $storyTitleArray = explode("/", $storyTitle); + $story = $storyTitleArray[0]; + $title = $storyTitleArray[1]; + $message .= "Story: '{$story}' Title: '{$title}' in Tests {$tests}\n\n"; + + } + throw new XmlException($message); + } + } + /** * This method transforms Magento severity values from Severity annotation * Returns Allure annotation value diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php index 5e58c4d53..0dcdedd7d 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php @@ -67,6 +67,15 @@ public function __construct() $this->modulePathExtractor = new ModulePathExtractor(); } + /** + * Getter for AnnotationExtractor + * @return AnnotationExtractor + */ + public function getAnnotationExtractor() + { + return $this->annotationExtractor; + } + /** * This method takes and array of test data and strips away irrelevant tags. The data is converted into an array of * TestObjects. @@ -102,7 +111,10 @@ public function extractTestData($testData) ); if (array_key_exists(self::TEST_ANNOTATIONS, $testData)) { - $testAnnotations = $this->annotationExtractor->extractAnnotations($testData[self::TEST_ANNOTATIONS]); + $testAnnotations = $this->annotationExtractor->extractAnnotations( + $testData[self::TEST_ANNOTATIONS], + $testData[self::NAME] + ); } //Override features with module name if present, populates it otherwise