Re-work to support changing prefix/classification in test itself

(refs idaholab#16410)

python/MooseDocs/common/parse_settings.py

@@ -49,7 +49,7 @@ def match_settings(known, raw):
             value = False
         elif value.lower() == 'none':
             value = None
-        elif all([v.isdigit() for v in value]):
+        elif value and all([v.isdigit() for v in value]):
             value = float(value)
 
         if key in known:

python/MooseDocs/extensions/sqa.py

@@ -82,7 +82,6 @@ def defaultConfig():
         config['requirement-groups'] = (dict(), "Allows requirement group names to be changed")
         config['reports'] = (None, "Build SQA reports for dashboard creation.")
         config['add_run_exception_to_failure_analysis'] = (True, "Automatically include RunException tests in the 'FAILURE_ANALYSIS' collection.")
-        config['custom-requirements'] = (dict(), "Add the ability to read in hit files for creating custom requirements.")
 
         # Disable by default to allow for updates to applications
         config['active'] = (False, config['active'][1])
@@ -125,58 +124,14 @@ def preExecute(self):
         start = time.time()
         LOG.info("Gathering SQA requirement information...")
 
-        repos = self.get('repos')
-        for index, (category, info) in enumerate(self.get('categories').items(), 1):
-            prefix = info.get('prefix', 'F')
-            specs = info.get('specs', ['tests'])
-            repo = info.get('repo', 'default')
-            reports = info.get('reports', None)
-
-            # Test directories
-            directories = list()
-            for d in info.get('directories', []):
-                path = mooseutils.eval_path(d)
-                if not os.path.isdir(path):
-                    path = os.path.join(MooseDocs.ROOT_DIR, d)
-                if not os.path.isdir(path):
-                    msg = "Input directory does not exist: %s"
-                    LOG.error(msg, path)
-                    continue
-                directories.append(path)
-
-            # Custom requirement files
-            files = list()
-            for f in info.get('files', []):
-                loc, fname = os.path.split(f)
-                path = os.path.join(mooseutils.eval_path(loc), fname)
-                if not os.path.isfile(path):
-                    path = os.path.join(MooseDocs.ROOT_DIR, loc, fname)
-                if not os.path.isfile(path):
-                    msg = "Input file does not exist: %s"
-                    LOG.error(msg, path)
-                    continue
-                files.append(path)
-
-            # Create requirement database from tests
-            self.__requirements[prefix][category].update(moosesqa.get_requirements_from_tests(directories, specs))
-
-            # Create requirement database from files
-            self.__requirements[prefix][category].update(moosesqa.get_requirements_from_files(files))
-
-            # Create dependency database
-            self.__dependencies[category] = info.get('dependencies', [])
-
-            # Create remote repository database
-            self.__remotes[category] = repos.get(repo, None)
-
-            # Create reports from included SQA apps (e.g., moose/modulus/stochastic_tools)
-            if reports:
-                self.__reports[category] = moosesqa.get_sqa_reports(reports)
+        # Build the requirements, dependencies, remotes, and reports data structures
+        for category, info in self.get('categories').items():
+            self._updateCategoryItems(category, info)
 
         # Number the requirements
-        for p in self.__requirements.keys():
-            for c, category in enumerate(self.__requirements[p].keys()):
-                moosesqa.number_requirements(self.__requirements[p][category], p, c+1)
+        for c, category in enumerate(self.__requirements.keys(), start=1):
+            for p, req_dict in self.__requirements[category].items():
+                moosesqa.number_requirements(req_dict, p, c)
 
         # Report for the entire app (e.g, moose/modules/doc)
         general_reports = self.get('reports')
@@ -208,37 +163,71 @@ def preExecute(self):
                     LOG.warning(msg)
 
         # Add RunException tests to FAILURE_ANALYSIS collection
-        for req_category in self.__requirements['F'].values():
-            for requirements in req_category.values():
-                for req in requirements:
-                    t_types = req.types
-                    if (req.collections is None) and (t_types is not None) and ('RunException' in t_types):
-                        req.collections = set(['FAILURE_ANALYSIS'])
+        for req_category in self.__requirements.values():
+            for req_prefix in req_category.values():
+                for requirements in req_prefix.values():
+                    for req in requirements:
+                        t_types = req.types
+                        if (req.collections is None) and (t_types is not None) and ('RunException' in t_types):
+                            req.collections = set(['FAILURE_ANALYSIS'])
 
         LOG.info("Gathering SQA requirement information complete [%s sec.]", time.time() - start)
 
-    def postTokenize(self, page, ast):
-        """Update the requirement storage with requirements given in input text."""
-        for key, value in self.getAttributeItems():
-            self.__requirements[key[0]][key[1]][key[2]].append(value)
+    def _updateCategoryItems(self, category, info):
+        """
+        Helper for processing 'categories' configuration items.
+        """
+        specs = info.get('specs', ['tests'])
+        repo = info.get('repo', 'default')
+        reports = info.get('reports', None)
+
+        # Test directories
+        directories = list()
+        for d in info.get('directories', []):
+            path = mooseutils.eval_path(d)
+            if not os.path.isdir(path):
+                path = os.path.join(MooseDocs.ROOT_DIR, d)
+            if not os.path.isdir(path):
+                msg = "Input directory does not exist: %s"
+                LOG.error(msg, path)
+                continue
+            directories.append(path)
+
+        # Create requirement database from tests
+        for group, requirements in moosesqa.get_requirements_from_tests(directories, specs).items():
+            for req in requirements:
+                key = req.classification or 'FUNCTIONAL'
+                if key not in moosesqa.MOOSESQA_CLASSIFICATION:
+                    self.setActive(False)
+                    msg = "SQA extension disabled.\nUnknown classification key in requirement: %s:%s"
+                    LOG.error(msg, req.filename, req.classification_line)
+                prefix = moosesqa.MOOSESQA_CLASSIFICATION[key][0]
+                self.__requirements[category][prefix][group].append(req)
+
+        # Create dependency database
+        self.__dependencies[category] = info.get('dependencies', [])
+
+        # Create remote repository database
+        repos = self.get('repos')
+        self.__remotes[category] = repos.get(repo, None)
 
-        # Re-number to account for custom requirements
-        for prefix, categories in self.__requirements.items():
-            for c, out in enumerate(categories.values()):
-                for i, requirements in enumerate(out.values()):
-                    for j, req in enumerate(requirements):
-                        req.label = "{}{}.{}.{}".format(prefix, c+1, i+1, j+1)
+        # Create reports from included SQA apps (e.g., moose/modulus/stochastic_tools)
+        if reports:
+            self.__reports[category] = moosesqa.get_sqa_reports(reports)
 
     def hasCivetExtension(self):
         """Return True if the CivetExtension exists."""
         return self.__has_civet
 
     def requirements(self, category, prefix='F'):
         """Return the requirements dictionaries."""
-        req = self.__requirements[prefix].get(category, None)
-        if req is None:
+        c = self.__requirements.get(category, None)
+        if c is None:
             raise exceptions.MooseDocsException("Unknown or missing 'category': {}", category)
-        return req
+        r = c.get(prefix, None)
+        if r is None:
+            raise exceptions.MooseDocsException("Unknown or missing 'prefix': {}", prefix)
+        return r
 
     def dependencies(self, category):
         """Return the dependency list for given category."""
@@ -333,6 +322,9 @@ def defaultSettings():
 
     def createToken(self, parent, info, page):
         category = self.settings['category']
+        if category == '_empty_':
+            return parent
+
         prefix = self.settings['prefix']
         group_map = self.extension.get('requirement-groups', dict())
         for group, requirements in self.extension.requirements(category, prefix).items():
@@ -459,8 +451,11 @@ def defaultSettings():
         return config
 
     def createToken(self, parent, info, page):
-        design = collections.defaultdict(list)
         category = self.settings['category']
+        if category == '_empty_':
+            return parent
+
+        design = collections.defaultdict(list)
         prefix = self.settings['prefix']
         for requirements in self.extension.requirements(category, prefix).values():
             for req in requirements:
@@ -500,9 +495,12 @@ def defaultSettings():
         return config
 
     def createToken(self, parent, info, page):
+        category = self.settings['category']
+        if category == '_empty_':
+            return parent
+
         subcommand = info['subcommand']
         self.settings['link-{}'.format(subcommand)] = True
-        category = self.settings['category']
         prefix = self.settings['prefix']
         matrix = SQARequirementMatrix(parent)
         SQARequirementMatrixHeading(matrix, category=category, string=subcommand.title())
@@ -525,8 +523,11 @@ def defaultSettings():
         return config
 
     def createToken(self, parent, info, page):
-        self.settings['link-collections'] = True
         category = self.settings['category']
+        if category == '_empty_':
+            return parent
+
+        self.settings['link-collections'] = True
         prefix = self.settings['prefix']
         collection_map = collections.defaultdict(list)
         allowed = self.settings['items']
@@ -583,7 +584,7 @@ def defaultSettings():
 
     def createToken(self, parent, info, page):
         suffix = self.settings['suffix']
-        category = self.settings['category']
+        category = self.settings['category'] or None
         if category == '_empty_':
             depends = self.extension.get('categories').keys()
         else:
@@ -610,7 +611,10 @@ def defaultSettings():
         return config
 
     def createToken(self, parent, info, page):
-        category = self.settings.get('category')
+        category = self.settings['category']
+        if category == '_empty_':
+            return parent
+
         suffix = self.settings.get('suffix')
         return autolink.AutoLink(parent, page='sqa/{}_{}.md'.format(category, suffix),
                                  optional=True, warning=True)

python/MooseDocs/extensions/template.py

@@ -72,7 +72,7 @@ def postTokenize(self, page, ast):
 
         unknown_items = items.difference(fields)
         if unknown_items:
-            msg = "Unknown template item(s): {}".format(', '.join(unknown_items))
+            msg = "Unknown template item(s): {}\n{}".format(', '.join(unknown_items), page.source)
             raise exceptions.MooseDocsException(msg)
 
 class TemplateLoadCommand(command.CommandComponent):

python/MooseDocs/test/content/extensions/sqa.md

@@ -36,14 +36,14 @@
 
 ## Custom Matrix
 
-!sqa requirements category=custom prefix=C
+!sqa requirements category=demo prefix=C
 
 ## Collections
 
 ### Collection List
 
 !sqa collections-list
 
-### Collections
+### Requirement Collection
 
 !sqa collections category=demo

python/MooseDocs/test/materialize.yml

@@ -62,10 +62,6 @@ Extensions:
                     - python/MooseDocs/test
                 specs:
                     - demo
-            custom:
-                prefix: C
-                files:
-                    - python/MooseDocs/test/custom.hit
 
     MooseDocs.extensions.civet:
         active: False

python/MooseDocs/test/moosedocs.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+#* This file is part of the MOOSE framework
+#* https://www.mooseframework.org
+#*
+#* All rights reserved, see COPYRIGHT for full restrictions
+#* https://github.com/idaholab/moose/blob/master/COPYRIGHT
+#*
+#* Licensed under LGPL 2.1, please see LICENSE for details
+#* https://www.gnu.org/licenses/lgpl-2.1.html
+
+import sys
+import os
+
+# Locate MOOSE directory, the default is the location of MOOSE as determined by this file, the realpath follows symlinks
+MOOSE_DIR = os.getenv('MOOSE_DIR', os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..')))
+if not os.path.exists(MOOSE_DIR):
+    print('Failed to locate MOOSE, specify the MOOSE_DIR environment variable.')
+    sys.exit(1)
+os.environ['MOOSE_DIR'] = MOOSE_DIR
+
+# Append MOOSE python directory
+MOOSE_PYTHON_DIR = os.path.join(MOOSE_DIR, 'python')
+if MOOSE_PYTHON_DIR not in sys.path:
+    sys.path.append(MOOSE_PYTHON_DIR)
+
+# Always operate from the directory of them moosedocs.py script, this allows the script to be
+# run from any location. CIVET runs 'modules/doc/moosedocs.py check' from the root directory.
+os.chdir(os.path.abspath(os.path.dirname(__file__)))
+
+import moosesqa
+moosesqa.MOOSESQA_CLASSIFICATION['CUSTOM'] = ('C', "This is a custom classification")
+
+from MooseDocs import main
+if __name__ == '__main__':
+    sys.exit(main.run())

python/MooseDocs/test/tree/demo

@@ -1,6 +1,7 @@
 [Tests]
   issues = '#1980'
   design = core.md
+  classification = custom
   [r0]
     requirement = "Tree One"
   []

python/TestHarness/testers/Tester.py

@@ -105,6 +105,7 @@ def validParams():
         params.addParam("verification", False, "Set to True to mark test as a verification problem.")
         params.addParam("deprecated", False, "When True the test is no longer considered part SQA process and as such does not include the need for a requirement definition.")
         params.addParam("collections", [], "A means for defining a collection of tests for SQA process.")
+        params.addParam("classification", 'functional', "A means for defining a requirement classification for SQA process.")
         return params
 
     # This is what will be checked for when we look for valid testers

python/moosesqa/Requirement.py

@@ -47,6 +47,8 @@ def __str__(self):
 @mooseutils.addProperty('design_line', ptype=int)
 @mooseutils.addProperty('collections', ptype=set)
 @mooseutils.addProperty('collections_line', ptype=int)
+@mooseutils.addProperty('classification', ptype=str)
+@mooseutils.addProperty('classification_line', ptype=int)
 @mooseutils.addProperty('deprecated', ptype=bool, default=False)
 @mooseutils.addProperty('deprecated_line', ptype=int)
 @mooseutils.addProperty('verification', ptype=list)

python/moosesqa/__init__.py

@@ -6,7 +6,7 @@
 
 from .get_sqa_reports import get_sqa_reports
 from .check_syntax import check_syntax, file_is_stub, find_md_file
-from .get_requirements import get_requirements_from_tests, get_requirements_from_file, get_requirements_from_files
+from .get_requirements import get_requirements_from_tests, get_requirements_from_file
 from .get_requirements import number_requirements, get_test_specification
 from .check_requirements import check_requirements
 from .SQAReport import SQAReport
@@ -20,3 +20,9 @@
 MOOSESQA_COLLECTIONS['FAILURE_ANALYSIS'] = "Requirements that perform check for simulation " \
                                            "integrity such as error handling and convergence " \
                                            "failures."
+
+MOOSESQA_CLASSIFICATION = dict()
+MOOSESQA_CLASSIFICATION['FUNCTIONAL'] = ('F', "Requirements that define uses cases and are correct, unambiguous, complete, consistent, verifiable, and traceable.")
+MOOSESQA_CLASSIFICATION['USABILITY'] = ('U', "Requirements for the system that include measurable effectiveness, efficiency, and satisfaction criteria in specific contexts of use.")
+MOOSESQA_CLASSIFICATION['PERFORMANCE'] = ('P', "Requirements to define the critical performance conditions and associated capabilities.")
+MOOSESQA_CLASSIFICATION['SYSTEM'] = ('S', "Requirements for interfaces among system elements and with external entities.")