MQE-1073: Mandatory Annotations

dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/AnnotationExtractorTest.php

@@ -0,0 +1,133 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+namespace tests\unit\Magento\FunctionalTestFramework\Test\Util;
+
+use AspectMock\Proxy\Verifier;
+use AspectMock\Test as AspectMock;
+use Magento\FunctionalTestingFramework\Test\Util\AnnotationExtractor;
+use Monolog\Handler\TestHandler;
+use Monolog\Logger;
+use PHPUnit\Framework\TestCase;
+use tests\unit\Util\TestLoggingUtil;
+
+class AnnotationExtractorTest extends TestCase
+{
+    /**
+     * Before test functionality
+     * @return void
+     */
+    public function setUp()
+    {
+        TestLoggingUtil::getInstance()->setMockLoggingUtil();
+    }
+
+    /**
+     * Annotation extractor takes in raw array and condenses it to expected format
+     *
+     * @throws \Exception
+     */
+    public function testExtractAnnotations()
+    {
+        // Test Data
+        $testAnnotations = [
+            "nodeName" => "annotations",
+            "features" => [
+                [
+                    "nodeName" => "features",
+                    "value" => "TestFeatures"
+                ]
+            ],
+            "stories" => [
+                [
+                    "nodeName" => "stories",
+                    "value" => "TestStories"
+                ]
+            ],
+            "description" => [
+                [
+                    "nodeName" => "description",
+                    "value" => "TestDescription"
+                ]
+            ],
+            "severity" => [
+                [
+                    "nodeName" => "severity",
+                    "value" => "CRITICAL"
+                ]
+            ],
+            "group" => [
+                [
+                    "nodeName" => "group",
+                    "value" => "TestGroup"
+                ]
+            ],
+        ];
+        // Perform Test
+        $extractor = new AnnotationExtractor();
+        $returnedAnnotations = $extractor->extractAnnotations($testAnnotations, "testFileName");
+
+        // Asserts
+
+        $this->assertEquals("TestFeatures", $returnedAnnotations['features'][0]);
+        $this->assertEquals("TestStories", $returnedAnnotations['stories'][0]);
+        $this->assertEquals("TestDescription", $returnedAnnotations['description'][0]);
+        $this->assertEquals("CRITICAL", $returnedAnnotations['severity'][0]);
+        $this->assertEquals("TestGroup", $returnedAnnotations['group'][0]);
+    }
+
+    /**
+     * Annotation extractor should throw warning when required annotations are missing
+     *
+     * @throws \Exception
+     */
+    public function testMissingAnnotations()
+    {
+        // Test Data, missing title, description, and severity
+        $testAnnotations = [
+            "nodeName" => "annotations",
+            "features" => [
+                [
+                    "nodeName" => "features",
+                    "value" => "TestFeatures"
+                ]
+            ],
+            "stories" => [
+                [
+                    "nodeName" => "stories",
+                    "value" => "TestStories"
+                ]
+            ],
+            "group" => [
+                [
+                    "nodeName" => "group",
+                    "value" => "TestGroup"
+                ]
+            ],
+        ];
+        // Perform Test
+        $extractor = new AnnotationExtractor();
+        $returnedAnnotations = $extractor->extractAnnotations($testAnnotations, "testFileName");
+
+        // Asserts
+        TestLoggingUtil::getInstance()->validateMockLogStatement(
+            'warning',
+            'Test testFileName is missing required annotations.',
+            [
+                'testName' => 'testFileName',
+                'missingAnnotations' => "title, description, severity"
+            ]
+        );
+    }
+
+    /**
+     * After class functionality
+     * @return void
+     */
+    public static function tearDownAfterClass()
+    {
+        TestLoggingUtil::getInstance()->clearMockLoggingUtil();
+    }
+}

src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php

@@ -5,6 +5,7 @@
  */
 namespace Magento\FunctionalTestingFramework\Test\Handlers;
 
+use Magento\FunctionalTestingFramework\Exceptions\Collector\ExceptionCollector;
 use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
 use Magento\FunctionalTestingFramework\Exceptions\XmlException;
 use Magento\FunctionalTestingFramework\ObjectManager\ObjectHandlerInterface;
@@ -136,13 +137,19 @@ private function initTestData()
             return;
         }
 
+        $exceptionCollector = new ExceptionCollector();
         foreach ($parsedTestArray[TestObjectHandler::XML_ROOT] as $testName => $testData) {
             if (!is_array($testData)) {
                 continue;
             }
-
-            $this->tests[$testName] = $testObjectExtractor->extractTestData($testData);
+            try {
+                $this->tests[$testName] = $testObjectExtractor->extractTestData($testData);
+            } catch (XmlException $exception) {
+                $exceptionCollector->addError(self::class, $exception->getMessage());
+            }
         }
+        $exceptionCollector->throwException();
+
         $testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness();
     }
 

src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php

@@ -7,6 +7,7 @@
 namespace Magento\FunctionalTestingFramework\Test\Util;
 
 use Magento\FunctionalTestingFramework\Exceptions\XmlException;
+use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil;
 
 /**
  * Class AnnotationExtractor
@@ -28,6 +29,12 @@ class AnnotationExtractor extends BaseObjectExtractor
         "AVERAGE" => "MINOR",
         "MINOR" => "TRIVIAL"
     ];
+    const REQUIRED_ANNOTATIONS = [
+        "stories",
+        "title",
+        "description",
+        "severity"
+    ];
 
     /**
      * AnnotationExtractor constructor.
@@ -67,7 +74,11 @@ public function extractAnnotations($testAnnotations, $filename)
             }
             $annotationObjects[$annotationKey] = $annotationValues;
         }
+
+        $this->validateMissingAnnotations($annotationObjects, $filename);
+
         $this->addStoryTitleToMap($annotationObjects, $filename);
+
         return $annotationObjects;
     }
 
@@ -86,6 +97,30 @@ public function addStoryTitleToMap($annotations, $filename)
         }
     }
 
+    /**
+     * Validates given annotations against list of required annotations.
+     * @param array $annotationObjects
+     * @return void
+     */
+    private function validateMissingAnnotations($annotationObjects, $filename)
+    {
+        $missingAnnotations = [];
+
+        foreach (self::REQUIRED_ANNOTATIONS as $REQUIRED_ANNOTATION) {
+            if (!array_key_exists($REQUIRED_ANNOTATION, $annotationObjects)) {
+                $missingAnnotations[] = $REQUIRED_ANNOTATION;
+            }
+        }
+
+        if (!empty($missingAnnotations)) {
+            $message = "Test {$filename} is missing required annotations.";
+            LoggingUtil::getInstance()->getLogger(ActionObject::class)->warning(
+                $message,
+                ["testName" => $filename, "missingAnnotations" => implode(", ", $missingAnnotations)]
+            );
+        }
+    }
+
     /**
      * Validates that all Story/Title combinations are unique, builds list of violators if found.
      * @throws XmlException

src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php

@@ -110,12 +110,10 @@ public function extractTestData($testData)
             'extends'
         );
 
-        if (array_key_exists(self::TEST_ANNOTATIONS, $testData)) {
-            $testAnnotations = $this->annotationExtractor->extractAnnotations(
-                $testData[self::TEST_ANNOTATIONS],
-                $testData[self::NAME]
-            );
-        }
+        $testAnnotations = $this->annotationExtractor->extractAnnotations(
+            $testData[self::TEST_ANNOTATIONS] ?? [],
+            $testData[self::NAME]
+        );
 
         //Override features with module name if present, populates it otherwise
         $testAnnotations["features"] = [$module];