sanitycheck: Add a feature which can handle pytest script.

1. Add a class of Pytest(Harness) which can handle pytest script
in harness.py
2. Use running_dir to store current test directory which be
used in pytest scrpt.
3. Add a new section(handle_script) which used to specify the pytest
script in yaml file.
4. Add usage of this feature into zephyr doc.

Signed-off-by: YouhuaX Zhu <youhuax.zhu@intel.com>

doc/guides/test/twister.rst

@@ -329,7 +329,7 @@ harness: <string>
     simple as a loopback wiring or a complete hardware test setup for
     sensor and IO testing.
     Usually pertains to external dependency domains but can be anything such as
-    console, sensor, net, keyboard, or Bluetooth.
+    console, sensor, net, keyboard, Bluetooth or pytest.
 
 harness_config: <harness configuration options>
     Extra harness configuration options to be used to select a board and/or
@@ -369,6 +369,11 @@ harness_config: <harness configuration options>
 
         Only one fixture can be defined per testcase.
 
+    handle_script: <pytest dirctory> (default pytest)
+        Specify a pytest directory which need to excute when test case begin to running,
+        default pytest directory name is pytest, after pytest finished, twister will
+        check if this case pass or fail according the pytest report.
+
     The following is an example yaml file with a few harness_config options.
 
     ::
@@ -390,6 +395,19 @@ harness_config: <harness configuration options>
              tags: sensors
              depends_on: i2c
 
+    The following is an example yaml file with pytest harness_config options, default pytest_root
+    name will be used if pytest_root not specified and the default pytest_timeout is 15 seconds.
+    as an example, test case in test/pytest_sample/ use the default value of these two options.
+
+    ::
+
+        tests:
+          pytest.example:
+            harness: pytest
+            harness_config:
+              pytest_root: [pytest directory name]
+              pytest_timeout: [timeout]
+
 filter: <expression>
     Filter whether the testcase should be run by evaluating an expression
     against an environment containing the following values:

scripts/pylib/twister/harness.py

@@ -1,8 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 import re
+import os
+import subprocess
 from collections import OrderedDict
+import xml.etree.ElementTree as ET
 
 result_re = re.compile(".*(PASS|FAIL|SKIP) - (test_)?(.*)")
+pytest_running_dir = ""
 
 class Harness:
     GCOV_START = "GCOV_COVERAGE_DUMP_START"
@@ -28,6 +32,10 @@ def __init__(self):
         self.recording = []
         self.fieldnames = []
         self.ztest = False
+        self.running_dir = None
+        self.source_dir = None
+        self.pytest_root = 'pytest'
+        self.pytest_timeout = 15
 
     def configure(self, instance):
         config = instance.testcase.harness_config
@@ -41,6 +49,10 @@ def configure(self, instance):
             self.repeat = config.get('repeat', 1)
             self.ordered = config.get('ordered', True)
             self.record = config.get('record', {})
+            self.running_dir = config.get('running_dir', None)
+            self.source_dir = config.get('source_dir', None)
+            self.pytest_root = config.get('pytest_root', 'pytest')
+            self.pytest_timeout = config.get('pytest_timeout', 15)
 
     def process_test(self, line):
 
@@ -121,6 +133,63 @@ def handle(self, line):
 
         self.process_test(line)
 
+class Pytest(Harness):
+
+    def handle(self, line):
+        ''' To keep artifacts of pytest in self.running_dir, pass this directory
+            by "--cmdopt". On pytest end, add a command line option and provide
+            the cmdopt through a fixture function
+			pytest handle output as a whole, it's not necessary to handle it
+			line by line
+        '''
+        global pytest_running_dir
+        if self.running_dir is pytest_running_dir:
+            return
+
+        pytest_running_dir = self.running_dir
+        cmd = [
+			'pytest',
+			'-s',
+			os.path.join(self.source_dir, self.pytest_root),
+			'--cmdopt',
+			self.running_dir,
+			'--junit-xml',
+			os.path.join(self.running_dir, 'report.html'),
+			'-q'
+        ]
+
+        with subprocess.Popen(cmd) as proc:
+            try:
+                proc.wait(self.pytest_timeout)
+                tree = ET.parse(os.path.join(self.running_dir, "report.html"))
+                root = tree.getroot()
+                for child in root:
+                    if child.tag == 'testsuite':
+                        if child.attrib['failures'] != '0':
+                            self.state = "failed"
+                        elif child.attrib['skipped'] != '0':
+                            self.state = "skipped"
+                        elif child.attrib['errors'] != '0':
+                            self.state = "errors"
+                        else:
+                            self.state = "passed"
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                self.state = "failed"
+            except ET.ParseError:
+                self.state = "failed"
+            except IOError:
+                self.state = "failed"
+
+        if self.state == "passed":
+            self.tests[self.id] = "PASS"
+        elif self.state == "skipped":
+            self.tests[self.id] = "SKIP"
+        else:
+            self.tests[self.id] = "FAIL"
+
+        self.process_test(line)
+
 class Test(Harness):
     RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL"
     RUN_FAILED = "PROJECT EXECUTION FAILED"

scripts/pylib/twister/twisterlib.py

@@ -509,6 +509,8 @@ def handle(self):
         harness_import = HarnessImporter(harness_name)
         harness = harness_import.instance
         harness.configure(self.instance)
+        harness.running_dir = self.build_dir
+        harness.source_dir = self.sourcedir
 
         if self.call_make_run:
             command = [self.generator_cmd, "run"]
@@ -789,6 +791,8 @@ def handle(self):
         harness_import = HarnessImporter(harness_name)
         harness = harness_import.instance
         harness.configure(self.instance)
+        harness.running_dir = self.build_dir
+        harness.source_dir = self.sourcedir
         read_pipe, write_pipe = os.pipe()
         start_time = time.time()
 
@@ -1053,6 +1057,9 @@ def handle(self):
         harness_import = HarnessImporter(self.instance.testcase.harness.capitalize())
         harness = harness_import.instance
         harness.configure(self.instance)
+        harness.running_dir = self.build_dir
+        harness.source_dir = self.sourcedir
+
         self.thread = threading.Thread(name=self.name, target=QEMUHandler._thread,
                                        args=(self, self.timeout, self.build_dir,
                                              self.log_fn, self.fifo_fn,
@@ -1757,7 +1764,7 @@ def __lt__(self, other):
     def testcase_runnable(testcase, fixtures):
         can_run = False
         # console harness allows us to run the test and capture data.
-        if testcase.harness in [ 'console', 'ztest']:
+        if testcase.harness in [ 'console', 'ztest', 'pytest']:
             can_run = True
             # if we have a fixture that is also being supplied on the
             # command-line, then we need to run the test, not just build it.

scripts/schemas/twister/testcase-schema.yaml

@@ -62,6 +62,12 @@ mapping:
           "repeat":
             type: int
             required: no
+          "pytest_root":
+            type: str
+            required: no
+          "pytest_timeout":
+            type: int
+            required: no
           "regex":
             type: seq
             required: no
@@ -187,6 +193,12 @@ mapping:
               "repeat":
                 type: int
                 required: no
+              "pytest_root":
+                type: str
+                required: no
+              "pytest_timeout":
+                type: int
+                required: no
               "regex":
                 type: seq
                 required: no