Initial custom parser implementation (#1283).

The code works and the code itself and the new API also look pretty
good for me. There is, however, plenty of work still:

- Acceptance tests. My initial tests work well, but they neeed to be
  converted to proper acceptance tests.

- Documentation in source (--help, API docs, ...). I added initial API
  docs but there are some holes and --help text is completely
  missing. I added FIXMEs to places where more docs are needed.

- User Guide documentation.

- `Defaults` API. We pass `Defaults` as the second argument to `parse`
  and `parse_init`, but that class itself doesn't have too good API.
  It needs to be enhanced and documented (incl. types) properly. The
  class could possibly also get a better name and it needs to be
  exposes directly via `robot.running`.

Although the feature is still work-in-progress, I commit it now to
make it possible for others to test the new API. It can the be still
enhanced based on feedback.

src/robot/api/interfaces.py

@@ -13,14 +13,15 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-"""Optional base classes for libraries and listeners.
+"""Optional base classes for libraries and other extensions.
 
 Module contents:
 
 - :class:`DynamicLibrary` for libraries using the `dynamic library API`__.
 - :class:`HybridLibrary` for libraries using the `hybrid library API`__.
 - :class:`ListenerV2` for `listener interface version 2`__.
 - :class:`ListenerV3` for `listener interface version 3`__.
+- :class:`Parser` for `custom parsers`__.
 - Type definitions used by the aforementioned classes.
 
 Main benefit of using these base classes is that editors can provide automatic
@@ -40,11 +41,13 @@
 __ http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#hybrid-library-api
 __ http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#listener-version-2
 __ http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#listener-version-3
+__ FIXME: PARSER: Link to UG docs.
 """
 
 import sys
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Tuple, Union
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 # Need to use version check and not try/except to support Mypy's stubgen.
 if sys.version_info >= (3, 8):
     from typing import TypedDict
@@ -57,6 +60,12 @@
 
 from robot import result, running
 from robot.model import Message
+from robot.running import TestSuite
+# FIXME: PARSER:
+# - Expose `Defaults` via `robot.running`.
+# - Consider better class name.
+# - Enhance its API (incl. docs and types).
+from robot.running.builder.settings import Defaults
 
 
 # Type aliases used by DynamicLibrary and HybridLibrary.
@@ -507,7 +516,7 @@ def close(self):
 
 
 class ListenerV3:
-    """Optional base class for listeners using the listener API v2."""
+    """Optional base class for listeners using the listener API v3."""
     ROBOT_LISTENER_API_VERSION = 3
 
     def start_suite(self, data: running.TestSuite, result: result.TestSuite):
@@ -560,3 +569,40 @@ def close(self):
 
         With library listeners called when the library goes out of scope.
         """
+
+
+class Parser(ABC):
+    """Optional base class for custom parsers.
+
+    Parsers do not need to explicitly extend this class and in simple cases
+    it is possible to implement them as modules. Regardless how a parser is
+    implemented, it must have :attr:`extension` attribute and :meth:`parse`
+    method. The :meth:`parse_init` method is optional and only needed if
+    a parser supports parsing suite initialization files.
+
+    The mandatory :attr:`extension` attribute specifies what file extension or
+    extensions a parser supports. It can be set either as a class or instance
+    attribute, and it can be either a string or a list/tuple of strings. The
+    attribute can also be named ``EXTENSION``, which typically works better
+    when a parser is implemented as a module.
+
+    The support for custom parsers is new in Robot Framework 6.1.
+    """
+    extension: Union[str, Sequence[str]]
+
+    @abstractmethod
+    def parse(self, source: Path, defaults: Defaults) -> TestSuite:
+        """Mandatory method for parsing suite files.
+
+        FIXME: PARSER: Better documentation (incl. parameter docs).
+        """
+        raise NotImplementedError
+
+    def parse_init(self, source: Path, defaults: Defaults) -> TestSuite:
+        """Optional method for parsing suite initialization files.
+
+        FIXME: PARSER: Better documentation (incl. parameter docs).
+
+        If not implemented, possible initialization files cause an error.
+        """
+        raise NotImplementedError

src/robot/conf/settings.py

@@ -53,8 +53,7 @@ class _BaseSettings:
                  'TimestampOutputs' : ('timestampoutputs', False),
                  'LogTitle'         : ('logtitle', None),
                  'ReportTitle'      : ('reporttitle', None),
-                 'ReportBackground' : ('reportbackground',
-                                       ('#9e9', '#f66', '#fed84f')),
+                 'ReportBackground' : ('reportbackground', ('#9e9', '#f66', '#fed84f')),
                  'SuiteStatLevel'   : ('suitestatlevel', -1),
                  'TagStatInclude'   : ('tagstatinclude', []),
                  'TagStatExclude'   : ('tagstatexclude', []),
@@ -470,6 +469,7 @@ class RobotSettings(_BaseSettings):
                        'RunEmptySuite'      : ('runemptysuite', False),
                        'Variables'          : ('variable', []),
                        'VariableFiles'      : ('variablefile', []),
+                       'Parsers'            : ('parser', []),
                        'PreRunModifiers'    : ('prerunmodifier', []),
                        'Listeners'          : ('listener', []),
                        'ConsoleType'        : ('console', 'verbose'),
@@ -629,6 +629,10 @@ def max_error_lines(self):
     def max_assign_length(self):
         return self['MaxAssignLength']
 
+    @property
+    def parsers(self):
+        return self['Parsers']
+
     @property
     def pre_run_modifiers(self):
         return self['PreRunModifiers']

src/robot/run.py

@@ -311,6 +311,7 @@
                           The seed must be an integer.
                           Examples: --randomize all
                                     --randomize tests:1234
+    --parser parser       FIXME: PARSER: Documentation
     --prerunmodifier class *  Class to programmatically modify the suite
                           structure before execution.
     --prerebotmodifier class *  Class to programmatically modify the result
@@ -413,8 +414,8 @@
 class RobotFramework(Application):
 
     def __init__(self):
-        Application.__init__(self, USAGE, arg_limits=(1,), env_options='ROBOT_OPTIONS',
-                             logger=LOGGER)
+        super().__init__(USAGE, arg_limits=(1,), env_options='ROBOT_OPTIONS',
+                         logger=LOGGER)
 
     def main(self, datasources, **options):
         try:
@@ -429,6 +430,7 @@ def main(self, datasources, **options):
             sys.path = settings.pythonpath + sys.path
         builder = TestSuiteBuilder(settings.suite_names,
                                    included_extensions=settings.extension,
+                                   custom_parsers=settings.parsers,
                                    rpa=settings.rpa,
                                    lang=settings.languages,
                                    allow_empty_suite=settings.run_empty_suite)

src/robot/running/builder/builders.py

@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+from itertools import chain
 from os.path import normpath
 from pathlib import Path
 from typing import Sequence
@@ -21,11 +22,11 @@
 from robot.errors import DataError
 from robot.output import LOGGER
 from robot.parsing import SuiteStructure, SuiteStructureBuilder, SuiteStructureVisitor
-from robot.utils import seq2str
+from robot.utils import Importer, seq2str, split_args_from_name_or_path
 
 from ..model import ResourceFile, TestSuite
-from .parsers import (JsonParser, NoInitFileDirectoryParser, Parser, RestParser,
-                      RobotParser)
+from .parsers import (CustomParser, JsonParser, NoInitFileDirectoryParser, Parser,
+                      RestParser, RobotParser)
 from .settings import Defaults
 
 
@@ -55,6 +56,7 @@ class TestSuiteBuilder:
 
     def __init__(self, included_suites: Sequence[str] = (),
                  included_extensions: Sequence[str] = ('.robot', '.rbt'),
+                 custom_parsers: Sequence[str] = (),
                  rpa: 'bool|None' = None, lang: LanguagesLike = None,
                  allow_empty_suite: bool = False, process_curdir: bool = True):
         """
@@ -63,6 +65,8 @@ def __init__(self, included_suites: Sequence[str] = (),
             Same as using `--suite` on the command line.
         :param included_extensions:
             List of extensions of files to parse. Same as `--extension`.
+        :param custom_parsers:
+            FIXME: PARSER: Documentation.
         :param rpa: Explicit execution mode. ``True`` for RPA and
             ``False`` for test automation. By default, mode is got from data file
             headers and possible conflicting headers cause an error.
@@ -80,31 +84,51 @@ def __init__(self, included_suites: Sequence[str] = (),
             resolved already at parsing time by default, but that can be
             changed by giving this argument ``False`` value.
         """
+        self.standard_parsers = self._get_standard_parsers(lang, process_curdir)
+        self.custom_parsers = self._get_custom_parsers(custom_parsers)
+        self.included_suites = tuple(included_suites or ())
+        self.included_extensions = tuple(included_extensions or ())
+        self.rpa = rpa
+        self.allow_empty_suite = allow_empty_suite
+
+    def _get_standard_parsers(self, lang: LanguagesLike,
+                              process_curdir: bool) -> 'dict[str, Parser]':
         robot_parser = RobotParser(lang, process_curdir)
         rest_parser = RestParser(lang, process_curdir)
         json_parser = JsonParser()
-        self.standard_parsers = {
+        return {
             'robot': robot_parser,
             'rst': rest_parser,
             'rest': rest_parser,
             'rbt': json_parser,
             'json': json_parser
         }
-        self.included_suites = tuple(included_suites or ())
-        self.included_extensions = tuple(included_extensions)
-        self.rpa = rpa
-        self.allow_empty_suite = allow_empty_suite
+
+    def _get_custom_parsers(self, names: Sequence[str]) -> 'dict[str, CustomParser]':
+        parsers = {}
+        importer = Importer('parser', LOGGER)
+        for name in names:
+            name, args = split_args_from_name_or_path(name)
+            imported = importer.import_class_or_module(name, args)
+            try:
+                parser = CustomParser(imported)
+            except TypeError as err:
+                raise DataError(f"Importing parser '{name}' failed: {err}")
+            for ext in parser.extensions:
+                parsers[ext] = parser
+        return parsers
 
     def build(self, *paths: 'Path|str'):
         """
         :param paths: Paths to test data files or directories.
         :return: :class:`~robot.running.model.TestSuite` instance.
         """
         paths = self._normalize_paths(paths)
-        parsers = self._get_parsers(self.included_extensions, paths)
-        structure = SuiteStructureBuilder(self.included_extensions,
+        extensions = chain(self.included_extensions, self.custom_parsers)
+        structure = SuiteStructureBuilder(extensions,
                                           self.included_suites).build(*paths)
-        suite = SuiteStructureParser(parsers, self.rpa).parse(structure)
+        suite = SuiteStructureParser(self._get_parsers(paths),
+                                     self.rpa).parse(structure)
         if not self.included_suites and not self.allow_empty_suite:
             self._validate_not_empty(suite, multi_source=len(paths) > 1)
         suite.remove_empty_suites(preserve_direct_children=len(paths) > 1)
@@ -124,12 +148,14 @@ def _normalize_paths(self, paths: 'tuple[Path|str]') -> 'tuple[Path]':
                             f"File or directory to execute does not exist.")
         return paths
 
-    def _get_parsers(self, extensions: 'tuple[str]', paths: 'tuple[Path]'):
-        parsers = {None: NoInitFileDirectoryParser()}
+    def _get_parsers(self, paths: 'tuple[Path]'):
+        parsers = {None: NoInitFileDirectoryParser(), **self.custom_parsers}
         robot_parser = self.standard_parsers['robot']
-        for ext in extensions + tuple(p.suffix for p in paths if p.is_file()):
+        for ext in chain(self.included_extensions,
+                         [p.suffix for p in paths if p.is_file()]):
             ext = ext.lstrip('.').lower()
-            parsers[ext] = self.standard_parsers.get(ext, robot_parser)
+            if ext not in parsers:
+                parsers[ext] = self.standard_parsers.get(ext, robot_parser)
         return parsers
 
     def _validate_not_empty(self, suite: TestSuite, multi_source: bool = False):

src/robot/running/builder/parsers.py

@@ -17,8 +17,9 @@
 from pathlib import Path
 
 from robot.conf import Languages
+from robot.errors import DataError
 from robot.parsing import File, get_init_model, get_model, get_resource_model
-from robot.utils import FileReader, read_rest_data
+from robot.utils import FileReader, get_error_message, read_rest_data
 
 from .settings import Defaults
 from .transformers import ResourceBuilder, SuiteBuilder
@@ -27,14 +28,18 @@
 
 class Parser(ABC):
 
+    @property
+    def name(self) -> str:
+        return type(self).__name__
+
     def parse_suite_file(self, source: Path, defaults: Defaults) -> TestSuite:
-        raise TypeError(f'{type(self).__name__} does not support suite files')
+        raise DataError(f"'{self.name}' does not support parsing suite files.")
 
     def parse_init_file(self, source: Path, defaults: Defaults) -> TestSuite:
-        raise TypeError(f'{type(self).__name__} does not support initialization files')
+        raise DataError(f"'{self.name}' does not support parsing initialization files.")
 
     def parse_resource_file(self, source: Path) -> ResourceFile:
-        raise TypeError(f'{type(self).__name__} does not support resource files')
+        raise DataError(f"'{self.name}' does not support parsing resource files.")
 
 
 class RobotParser(Parser):
@@ -102,3 +107,48 @@ class NoInitFileDirectoryParser(Parser):
 
     def parse_init_file(self, source: Path, defaults: Defaults) -> TestSuite:
         return TestSuite(name=TestSuite.name_from_source(source), source=source)
+
+
+class CustomParser(Parser):
+
+    def __init__(self, parser):
+        self.parser = parser
+        if not callable(getattr(parser, 'parse', None)):
+            raise TypeError(f"'{self.name}' does not have mandatory 'parse' method.")
+        if not self.extensions:
+            raise TypeError(f"'{self.name}' does not have mandatory 'EXTENSION' "
+                            f"or 'extension' attribute set.")
+
+    @property
+    def name(self) -> str:
+        return type(self.parser).__name__
+
+    @property
+    def extensions(self) -> 'tuple[str]':
+        ext = (getattr(self.parser, 'EXTENSION', ())
+               or getattr(self.parser, 'extension', ()))
+        extensions = [ext] if isinstance(ext, str) else tuple(ext)
+        return tuple(ext.lower().lstrip('.') for ext in extensions)
+
+    def parse_suite_file(self, source: Path, defaults: Defaults) -> TestSuite:
+        return self._parse(self.parser.parse, source, defaults)
+
+    def parse_init_file(self, source: Path, defaults: Defaults) -> TestSuite:
+        parse_init = getattr(self.parser, 'parse_init', None)
+        try:
+            return self._parse(parse_init, source, defaults)
+        except NotImplementedError:
+            return super().parse_init_file(source, defaults)    # Raises DataError
+
+    def _parse(self, method, *args) -> TestSuite:
+        if not method:
+            raise NotImplementedError
+        try:
+            suite = method(*args)
+            if not isinstance(suite, TestSuite):
+                raise TypeError(f"Return value should be 'robot.running.TestSuite', "
+                                f"got '{type(suite).__name__}'.")
+        except Exception:
+            raise DataError(f"Calling '{self.name}.{method.__name__}()' failed: "
+                            f"{get_error_message()}")
+        return suite