Replace validate-modules's semantic markup parser with antsibull-docs-parser

changelogs/fragments/80406-validate-modules-semantic-markup.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - "validate-modules sanity test - replace semantic markup parsing and validating code with the code from `antsibull-docs-parser 0.2.0 <https://github.com/ansible-community/antsibull-docs-parser/releases/tag/0.2.0>`__ (https://github.com/ansible/ansible/pull/80406)."

test/integration/targets/ansible-test-sanity-validate-modules/ansible_collections/ns/col/plugins/modules/semantic_markup.py

@@ -75,6 +75,23 @@
   a10:
     description: O(foo.bar=1).
     type: str
+
+  a11:
+    description: Something with suboptions.
+    type: dict
+    suboptions:
+      b1:
+        description:
+          - V(C\(foo\)).
+          - RV(bam).
+          - P(foo.bar#baz).
+          - P(foo.bar.baz).
+          - P(foo.bar.baz#woof).
+          - E(foo\(bar).
+          - O(bar).
+          - O(bar=bam).
+          - O(foo.bar=1).
+        type: str
 '''
 
 EXAMPLES = '''#'''
@@ -103,5 +120,8 @@
         a8=dict(),
         a9=dict(),
         a10=dict(),
+        a11=dict(type='dict', options=dict(
+            b1=dict(),
+        ))
     ))
     module.exit_json()

test/integration/targets/ansible-test-sanity-validate-modules/expected.txt

@@ -9,11 +9,16 @@ plugins/modules/invalid_yaml_syntax.py:0:0: missing-documentation: No DOCUMENTAT
 plugins/modules/invalid_yaml_syntax.py:8:15: documentation-syntax-error: DOCUMENTATION is not valid YAML
 plugins/modules/invalid_yaml_syntax.py:12:15: invalid-examples: EXAMPLES is not valid YAML
 plugins/modules/invalid_yaml_syntax.py:16:15: return-syntax-error: RETURN is not valid YAML
-plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a2.description: Directive "V(C\(foo\))" contains unnecessarily quoted "(" for dictionary value @ data['options']['a2']['description']. Got 'V(C\\(foo\\)).'
-plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a4.description: Directive "P(foo.bar#baz)" must contain a FQCN; found "foo.bar" for dictionary value @ data['options']['a4']['description']. Got 'P(foo.bar#baz).'
-plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a5.description: Directive "P(foo.bar.baz)" must contain a "#" for dictionary value @ data['options']['a5']['description']. Got 'P(foo.bar.baz).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a11.suboptions.b1.description.0: While parsing "V(C\(" at index 1: Unnecessarily escaped "(" @ data['options']['a11']['suboptions']['b1']['description'][0]. Got 'V(C\\(foo\\)).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a11.suboptions.b1.description.2: While parsing "P(foo.bar#baz)" at index 1: Plugin name "foo.bar" is not a FQCN @ data['options']['a11']['suboptions']['b1']['description'][2]. Got 'P(foo.bar#baz).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a11.suboptions.b1.description.3: While parsing "P(foo.bar.baz)" at index 1: Parameter "foo.bar.baz" is not of the form FQCN#type @ data['options']['a11']['suboptions']['b1']['description'][3]. Got 'P(foo.bar.baz).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a11.suboptions.b1.description.4: Directive "P(foo.bar.baz#woof)" must contain a valid plugin type; found "woof" @ data['options']['a11']['suboptions']['b1']['description'][4]. Got 'P(foo.bar.baz#woof).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a11.suboptions.b1.description.5: While parsing "E(foo\(" at index 1: Unnecessarily escaped "(" @ data['options']['a11']['suboptions']['b1']['description'][5]. Got 'E(foo\\(bar).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a2.description: While parsing "V(C\(" at index 1: Unnecessarily escaped "(" for dictionary value @ data['options']['a2']['description']. Got 'V(C\\(foo\\)).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a4.description: While parsing "P(foo.bar#baz)" at index 1: Plugin name "foo.bar" is not a FQCN for dictionary value @ data['options']['a4']['description']. Got 'P(foo.bar#baz).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a5.description: While parsing "P(foo.bar.baz)" at index 1: Parameter "foo.bar.baz" is not of the form FQCN#type for dictionary value @ data['options']['a5']['description']. Got 'P(foo.bar.baz).'
 plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a6.description: Directive "P(foo.bar.baz#woof)" must contain a valid plugin type; found "woof" for dictionary value @ data['options']['a6']['description']. Got 'P(foo.bar.baz#woof).'
-plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a7.description: Directive "E(foo\(bar)" contains unnecessarily quoted "(" for dictionary value @ data['options']['a7']['description']. Got 'E(foo\\(bar).'
+plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: DOCUMENTATION.options.a7.description: While parsing "E(foo\(" at index 1: Unnecessarily escaped "(" for dictionary value @ data['options']['a7']['description']. Got 'E(foo\\(bar).'
 plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: Directive "O(bar)" contains a non-existing option "bar"
 plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: Directive "O(bar=bam)" contains a non-existing option "bar"
 plugins/modules/semantic_markup.py:0:0: invalid-documentation-markup: Directive "O(foo.bar=1)" contains a non-existing option "foo.bar"

test/lib/ansible_test/_data/requirements/sanity.validate-modules.in

@@ -1,3 +1,4 @@
 jinja2  # ansible-core requirement
 pyyaml  # needed for collection_detail.py
 voluptuous
+antsibull-docs-parser==0.2.0

test/lib/ansible_test/_data/requirements/sanity.validate-modules.txt

@@ -1,4 +1,5 @@
 # edit "sanity.validate-modules.in" and generate with: hacking/update-sanity-requirements.py --test validate-modules
+antsibull-docs-parser==0.2.0
 Jinja2==3.1.2
 MarkupSafe==2.1.2
 PyYAML==6.0

test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py

@@ -33,6 +33,15 @@
 from contextlib import contextmanager
 from fnmatch import fnmatch
 
+try:
+    from antsibull_docs_parser import dom
+    from antsibull_docs_parser.parser import parse, Context
+except ImportError:
+    import traceback
+    ANTSIBULL_DOCS_PARSER_FAILURE = traceback.format_exc()
+else:
+    ANTSIBULL_DOCS_PARSER_FAILURE = None
+
 import yaml
 
 from voluptuous.humanize import humanize_error
@@ -79,10 +88,6 @@ def setup_collection_loader():
     ansible_module_kwargs_schema,
     doc_schema,
     return_schema,
-    _SEM_OPTION_NAME,
-    _SEM_RET_VALUE,
-    _check_sem_quoting,
-    _parse_prefix,
 )
 
 from .utils import CaptureStd, NoArgsAnsibleModule, compare_unordered_lists, parse_yaml, parse_isodate
@@ -1164,39 +1169,31 @@ def _validate_docs(self):
 
         return doc_info, doc
 
-    def _check_sem_option(self, directive, content):
-        try:
-            content = _check_sem_quoting(directive, content)
-            plugin_fqcn, plugin_type, option_link, option, value = _parse_prefix(directive, content)
-        except Exception:
-            # Validation errors have already been covered in the schema check
+    def _check_sem_option(self, part: dom.OptionNamePart, current_plugin: dom.PluginIdentifier) -> None:
+        if part.plugin is None or part.plugin != current_plugin:
             return
-        if plugin_fqcn is not None:
+        if part.entrypoint is not None:
             return
-        if tuple(option_link) not in self._all_options:
+        if tuple(part.link) not in self._all_options:
             self.reporter.error(
                 path=self.object_path,
                 code='invalid-documentation-markup',
-                msg='Directive "%s" contains a non-existing option "%s"' % (directive, option)
+                msg='Directive "%s" contains a non-existing option "%s"' % (part.source, part.name)
             )
 
-    def _check_sem_return_value(self, directive, content):
-        try:
-            content = _check_sem_quoting(directive, content)
-            plugin_fqcn, plugin_type, rv_link, rv, value = _parse_prefix(directive, content)
-        except Exception:
-            # Validation errors have already been covered in the schema check
+    def _check_sem_return_value(self, part: dom.ReturnValuePart, current_plugin: dom.PluginIdentifier) -> None:
+        if part.plugin is None or part.plugin != current_plugin:
             return
-        if plugin_fqcn is not None:
+        if part.entrypoint is not None:
             return
-        if tuple(rv_link) not in self._all_return_values:
+        if tuple(part.link) not in self._all_return_values:
             self.reporter.error(
                 path=self.object_path,
                 code='invalid-documentation-markup',
-                msg='Directive "%s" contains a non-existing return value "%s"' % (directive, rv)
+                msg='Directive "%s" contains a non-existing return value "%s"' % (part.source, part.name)
             )
 
-    def _validate_semantic_markup(self, object):
+    def _validate_semantic_markup(self, object) -> None:
         # Make sure we operate on strings
         if is_iterable(object):
             for entry in object:
@@ -1205,10 +1202,19 @@ def _validate_semantic_markup(self, object):
         if not isinstance(object, string_types):
             return
 
-        for m in _SEM_OPTION_NAME.finditer(object):
-            self._check_sem_option(m.group(0), m.group(1))
-        for m in _SEM_RET_VALUE.finditer(object):
-            self._check_sem_return_value(m.group(0), m.group(1))
+        if self.collection:
+            fqcn = f'{self.collection_name}.{self.name}'
+        else:
+            fqcn = f'ansible.builtin.{self.name}'
+        current_plugin = dom.PluginIdentifier(fqcn=fqcn, type=self.plugin_type)
+        for par in parse(object, Context(current_plugin=current_plugin), errors='message', add_source=True):
+            for part in par:
+                # Errors are already covered during schema validation, we only check for option and
+                # return value references
+                if part.type == dom.PartType.OPTION_NAME:
+                    self._check_sem_option(part, current_plugin)
+                if part.type == dom.PartType.RETURN_VALUE:
+                    self._check_sem_return_value(part, current_plugin)
 
     def _validate_semantic_markup_collect(self, destination, sub_key, data, all_paths):
         if not isinstance(data, dict):
@@ -2611,6 +2617,10 @@ def run():
 
 
 def main():
+    if ANTSIBULL_DOCS_PARSER_FAILURE is not None:
+        print('FATAL ERROR: Cannot import antsibull-docs-parser')
+        print(ANTSIBULL_DOCS_PARSER_FAILURE)
+        sys.exit(255)
     try:
         run()
     except KeyboardInterrupt: