Merge pull request #5897 from clebergnu/runnable_recipe_validation_re…

…solver

Signed-off-by: Jan Richter <jarichte@redhat.com>

MANIFEST.in

@@ -4,5 +4,6 @@ include README.rst
 include VERSION
 recursive-include avocado/etc *
 recursive-include avocado/libexec *
+recursive-include avocado/schemas *
 recursive-include selftests *
 recursive-include examples *

avocado/core/nrunner/runnable.py

@@ -2,11 +2,19 @@
 import collections
 import json
 import logging
+import os
 import subprocess
 import sys
 
 import pkg_resources
 
+try:
+    import jsonschema
+
+    JSONSCHEMA_AVAILABLE = True
+except ImportError:
+    JSONSCHEMA_AVAILABLE = False
+
 from avocado.core.nrunner.config import ConfigDecoder, ConfigEncoder
 from avocado.core.settings import settings
 from avocado.core.utils.eggenv import get_python_path_env_if_egg
@@ -20,6 +28,14 @@
 #: The configuration that is known to be used by standalone runners
 STANDALONE_EXECUTABLE_CONFIG_USED = {}
 
+#: Location used for schemas when packaged (as in RPMs)
+SYSTEM_WIDE_SCHEMA_PATH = "/usr/share/avocado/schemas"
+
+
+class RunnableRecipeInvalidError(Exception):
+    """Signals that a runnable recipe is not well formed, contains
+    missing or bad data"""
+
 
 def _arg_decode_base64(arg):
     """
@@ -196,6 +212,53 @@ def from_args(cls, args):
             **_key_val_args_to_kwargs(args.get("kwargs", [])),
         )
 
+    @staticmethod
+    def _validate_recipe_json_schema(recipe):
+        """Attempts to validate the runnable recipe using a JSON schema
+
+        :param recipe: the recipe already parsed from JSON into a dict
+        :type recipe: dict
+        :returns: whether the runnable recipe JSON was attempted to be
+                  validated with a JSON schema
+        :rtype: bool
+        :raises: RunnableRecipeInvalidError if the recipe is invalid
+        """
+        if not JSONSCHEMA_AVAILABLE:
+            return False
+        schema_filename = "runnable-recipe.schema.json"
+        schema_path = pkg_resources.resource_filename(
+            "avocado", os.path.join("schemas", schema_filename)
+        )
+        if not os.path.exists(schema_path):
+            schema_path = os.path.join(SYSTEM_WIDE_SCHEMA_PATH, schema_filename)
+            if not os.path.exists(schema_path):
+                return False
+        with open(schema_path, "r", encoding="utf-8") as schema:
+            try:
+                jsonschema.validate(recipe, json.load(schema))
+            except jsonschema.exceptions.ValidationError as details:
+                raise RunnableRecipeInvalidError(details)
+        return True
+
+    @classmethod
+    def _validate_recipe(cls, recipe):
+        """Validates a recipe using either JSON schema or builtin logic
+
+        :param recipe: the recipe already parsed from JSON into a dict
+        :type recipe: dict
+        :returns: None
+        :raises: RunnableRecipeInvalidError if the recipe is invalid
+        """
+        if not cls._validate_recipe_json_schema(recipe):
+            # This is a simplified validation of the recipe
+            allowed = set(["kind", "uri", "args", "kwargs", "config"])
+            if not "kind" in recipe:
+                raise RunnableRecipeInvalidError('Missing required property "kind"')
+            if not set(recipe.keys()).issubset(allowed):
+                raise RunnableRecipeInvalidError(
+                    "Additional properties are not allowed"
+                )
+
     @classmethod
     def from_recipe(cls, recipe_path):
         """
@@ -207,6 +270,7 @@ def from_recipe(cls, recipe_path):
         """
         with open(recipe_path, encoding="utf-8") as recipe_file:
             recipe = json.load(recipe_file)
+            cls._validate_recipe(recipe)
         config = ConfigDecoder.decode_set(recipe.get("config", {}))
         return cls.from_avocado_config(
             recipe.get("kind"),

avocado/core/output.py

@@ -105,8 +105,8 @@ def __init__(self):
                 self.disable()
         elif force_color != "always":
             raise ValueError(
-                "The value for runner.output.color must be one of "
-                "'always', 'never', 'auto' and not " + force_color
+                f"The value for runner.output.color must be one of "
+                f"'always', 'never', 'auto' and not {force_color}"
             )
 
     def disable(self):
@@ -134,87 +134,87 @@ def header_str(self, msg):
 
         If the output does not support colors, just return the original string.
         """
-        return self.HEADER + msg + self.ENDC
+        return f"{self.HEADER}{msg}{self.ENDC}"
 
     def fail_header_str(self, msg):
         """
         Print a fail header string (red colored).
 
         If the output does not support colors, just return the original string.
         """
-        return self.FAIL + msg + self.ENDC
+        return f"{self.FAIL}{msg}{self.ENDC}"
 
     def warn_header_str(self, msg):
         """
         Print a warning header string (yellow colored).
 
         If the output does not support colors, just return the original string.
         """
-        return self.WARN + msg + self.ENDC
+        return f"{self.WARN}{msg}{self.ENDC}"
 
     def healthy_str(self, msg):
         """
         Print a healthy string (green colored).
 
         If the output does not support colors, just return the original string.
         """
-        return self.PASS + msg + self.ENDC
+        return f"{self.PASS}{msg}{self.ENDC}"
 
     def partial_str(self, msg):
         """
         Print a string that denotes partial progress (yellow colored).
 
         If the output does not support colors, just return the original string.
         """
-        return self.PARTIAL + msg + self.ENDC
+        return f"{self.PARTIAL}{msg}{self.ENDC}"
 
     def pass_str(self, msg="PASS", move=MOVE_BACK):
         """
         Print a pass string (green colored).
 
         If the output does not support colors, just return the original string.
         """
-        return move + self.PASS + msg + self.ENDC
+        return f"{move}{self.PASS}{msg}{self.ENDC}"
 
     def skip_str(self, msg="SKIP", move=MOVE_BACK):
         """
         Print a skip string (yellow colored).
 
         If the output does not support colors, just return the original string.
         """
-        return move + self.SKIP + msg + self.ENDC
+        return f"{move}{self.SKIP}{msg}{self.ENDC}"
 
     def fail_str(self, msg="FAIL", move=MOVE_BACK):
         """
         Print a fail string (red colored).
 
         If the output does not support colors, just return the original string.
         """
-        return move + self.FAIL + msg + self.ENDC
+        return f"{move}{self.FAIL}{msg}{self.ENDC}"
 
     def error_str(self, msg="ERROR", move=MOVE_BACK):
         """
         Print a error string (red colored).
 
         If the output does not support colors, just return the original string.
         """
-        return move + self.ERROR + msg + self.ENDC
+        return f"{move}{self.ERROR}{msg}{self.ENDC}"
 
     def interrupt_str(self, msg="INTERRUPT", move=MOVE_BACK):
         """
         Print an interrupt string (red colored).
 
         If the output does not support colors, just return the original string.
         """
-        return move + self.INTERRUPT + msg + self.ENDC
+        return f"{move}{self.INTERRUPT}{msg}{self.ENDC}"
 
     def warn_str(self, msg="WARN", move=MOVE_BACK):
         """
         Print an warning string (yellow colored).
 
         If the output does not support colors, just return the original string.
         """
-        return move + self.WARN + msg + self.ENDC
+        return f"{move}{self.WARN}{msg}{self.ENDC}"
 
 
 #: Transparently handles colored terminal, when one is used
@@ -725,10 +725,10 @@ class Throbber:
     # Only print a throbber when we're on a terminal
     if TERM_SUPPORT.enabled:
         MOVES = [
-            TERM_SUPPORT.MOVE_BACK + STEPS[0],
-            TERM_SUPPORT.MOVE_BACK + STEPS[1],
-            TERM_SUPPORT.MOVE_BACK + STEPS[2],
-            TERM_SUPPORT.MOVE_BACK + STEPS[3],
+            f"{TERM_SUPPORT.MOVE_BACK}{STEPS[0]}",
+            f"{TERM_SUPPORT.MOVE_BACK}{STEPS[1]}",
+            f"{TERM_SUPPORT.MOVE_BACK}{STEPS[2]}",
+            f"{TERM_SUPPORT.MOVE_BACK}{STEPS[3]}",
         ]
     else:
         MOVES = ["", "", "", ""]

avocado/plugins/list.py

@@ -48,8 +48,8 @@ class List(CLICmd):
     def _prepare_matrix_for_display(matrix, verbose=False):
         colored_matrix = []
         for item in matrix:
-            cls = item[0]
-            type_label = TERM_SUPPORT.healthy_str(cls)
+            kind = item[0]
+            type_label = TERM_SUPPORT.healthy_str(kind)
             if verbose:
                 colored_matrix.append(
                     (type_label, item[1], _get_tags_as_string(item[2] or {}))
@@ -127,7 +127,11 @@ def _display_extra(suite, verbose=True):
 
     @staticmethod
     def _get_resolution_matrix(suite):
-        """Used for resolver."""
+        """Used for resolver.
+
+        :returns: a list of tuples with either (kind, uri, tags) or
+                  (kind, uri) depending on whether verbose mode is on
+        """
         test_matrix = []
         verbose = suite.config.get("core.verbose")
         for runnable in suite.tests:

avocado/plugins/resolvers.py

@@ -143,3 +143,20 @@ def resolve(self, reference):
         return ReferenceResolution(
             reference, ReferenceResolutionResult.SUCCESS, [runnable]
         )
+
+
+class RunnableRecipeResolver(Resolver):
+    name = "runnable-recipe"
+    description = "Test resolver for JSON runnable recipes"
+
+    def resolve(self, reference):
+        criteria_check = check_file(
+            reference, reference, suffix=".json", type_name="JSON file"
+        )
+        if criteria_check is not True:
+            return criteria_check
+
+        runnable = Runnable.from_recipe(reference)
+        return ReferenceResolution(
+            reference, ReferenceResolutionResult.SUCCESS, [runnable]
+        )

docs/source/guides/writer/chapters/recipes.rst

@@ -0,0 +1,59 @@
+Defining what to run using recipe files
+---------------------------------------
+
+If you've followed the previous documentation sections, you should now
+be able to write ``exec-test`` tests and also ``avocado-instrumented``
+tests.  These tests should be found when you run
+``avocado run /reference/to/a/test``.  Internally, though, these will
+be defined as a :class:`avocado.core.nrunner.runnable.Runnable`.
+
+This is interesting because you are able to have a shortcut into what
+Avocado runs by defining a ``Runnable``. Runnables can be defined using
+pure Python code, such as in the following Job example:
+
+.. literalinclude:: ../../../../../examples/jobs/custom_exec_test.py
+
+But, they can also be defined in JSON files, which we call "runnable
+recipes", such as:
+
+.. literalinclude:: ../../../../../examples/nrunner/recipes/runnables/exec_test_sleep_3.json
+
+
+Runnable recipe format
+~~~~~~~~~~~~~~~~~~~~~~
+
+While it should be somewhat easy to see the similarities between
+between the fields in the
+:class:`avocado.core.nrunner.runnable.Runnable` structure and a
+runnable recipe JSON data, Avocado actually ships with a schema that
+defines the exact format of the runnable recipe:
+
+.. literalinclude:: ../../../../../avocado/schemas/runnable-recipe.schema.json
+
+Avocado will attempt to enforce the JSON schema any time a
+``Runnable`` is loaded from such recipe files.
+
+Using runnable recipes as references
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Avocado ships with a ``runnable-recipe`` resolver plugin, which means
+that you can use runnable recipe file as a reference, and get
+something that Avocado can run (that is, a ``Runnable``).  Example::
+
+   avocado list examples/nrunner/recipes/runnables/python_unittest.json
+   python-unittest selftests/unit/test.py:TestClassTestUnit.test_long_name
+
+And just as runnable recipe's resolution can be listed, they can also
+be executed::
+
+  avocado run examples/nrunner/recipes/runnables/python_unittest.json
+  JOB ID     : bca087e0e5f16e62f24430602f87df67ecf093f7
+  JOB LOG    : ~/avocado/job-results/job-2024-04-17T11.53-bca087e/job.log
+   (1/1) selftests/unit/test.py:TestClassTestUnit.test_long_name: STARTED
+   (1/1) selftests/unit/test.py:TestClassTestUnit.test_long_name: PASS (0.02 s)
+  RESULTS    : PASS 1 | ERROR 0 | FAIL 0 | SKIP 0 | WARN 0 | INTERRUPT 0 | CANCEL 0
+  JOB TIME   : 2.72 s
+
+.. tip:: As a possible integration strategy with existing tests, you
+         can have one or more runnable recipe files that are passed
+         to Avocado to be executed.

docs/source/guides/writer/index.rst

@@ -7,6 +7,7 @@ Avocado Test Writer's Guide
 
    chapters/basics
    chapters/writing
+   chapters/recipes
    chapters/logging
    chapters/parameters
    chapters/libs

python-avocado.spec

@@ -28,7 +28,7 @@
 Summary: Framework with tools and libraries for Automated Testing
 Name: python-avocado
 Version: 104.0
-Release: 1%{?gitrel}%{?dist}
+Release: 2%{?gitrel}%{?dist}
 License: GPLv2+ and GPLv2 and MIT
 URL: https://avocado-framework.github.io/
 %if 0%{?rel_build}
@@ -84,6 +84,7 @@ Requires: python3-avocado-common == %{version}-%{release}
 Requires: gdb
 Requires: gdb-gdbserver
 Requires: procps-ng
+Requires: python3-jsonschema
 %if ! 0%{?rhel}
 Requires: python3-pycdlib
 %endif
@@ -191,9 +192,12 @@ cp -r examples/tests %{buildroot}%{_docdir}/avocado
 cp -r examples/yaml_to_mux %{buildroot}%{_docdir}/avocado
 cp -r examples/varianter_pict %{buildroot}%{_docdir}/avocado
 cp -r examples/varianter_cit %{buildroot}%{_docdir}/avocado
+mkdir -p %{buildroot}%{_datarootdir}/avocado
+mv %{buildroot}%{python3_sitelib}/avocado/schemas %{buildroot}%{_datarootdir}/avocado
 find %{buildroot}%{_docdir}/avocado -type f -name '*.py' -exec chmod -c -x {} ';'
 mkdir -p %{buildroot}%{_libexecdir}/avocado
 mv %{buildroot}%{python3_sitelib}/avocado/libexec/* %{buildroot}%{_libexecdir}/avocado
+rmdir %{buildroot}%{python3_sitelib}/avocado/libexec
 
 %if %{with tests}
 %check
@@ -263,6 +267,10 @@ Common files (such as configuration) for the Avocado Testing Framework.
 %dir %{_sysconfdir}/avocado/scripts/job/pre.d
 %dir %{_sysconfdir}/avocado/scripts/job/post.d
 %dir %{_sharedstatedir}/avocado
+%dir %{_sharedstatedir}/avocado/data
+%dir %{_datarootdir}/avocado
+%dir %{_datarootdir}/avocado/schemas
+%{_datarootdir}/avocado/schemas/*
 %config(noreplace)%{_sysconfdir}/avocado/sysinfo/commands
 %config(noreplace)%{_sysconfdir}/avocado/sysinfo/files
 %config(noreplace)%{_sysconfdir}/avocado/sysinfo/profilers
@@ -436,6 +444,12 @@ Again Shell code (and possibly other similar shells).
 %{_libexecdir}/avocado*
 
 %changelog
+* Tue Apr  2 2024 Cleber Rosa <crosa@redhat.com> - 104.0-2
+- Package JSON schema files
+- Removed empty libexec dir
+- Require python3-jsonschema to perform runtime schema validation
+  for recipe files
+
 * Tue Mar 19 2024 Jan Richter <jarichte@redhat.com> - 104.0-1
 - New release