sanitycheck: Add a feature which can handle pytest script.

doc/guides/test/twister.rst

@@ -329,7 +329,7 @@ harness: <string>
     simple as a loopback wiring or a complete hardware test setup for
     sensor and IO testing.
     Usually pertains to external dependency domains but can be anything such as
-    console, sensor, net, keyboard, or Bluetooth.
+    console, sensor, net, keyboard, Bluetooth or pytest.
 
 harness_config: <harness configuration options>
     Extra harness configuration options to be used to select a board and/or
@@ -369,6 +369,11 @@ harness_config: <harness configuration options>
 
         Only one fixture can be defined per testcase.
 
+    handle_script: <pytest dirctory> (default pytest)
+        Specify a pytest directory which need to excute when test case begin to running,
+        default pytest directory name is pytest, after pytest finished, twister will
+        check if this case pass or fail according the pytest report.
+
     The following is an example yaml file with a few harness_config options.
 
     ::
@@ -390,6 +395,19 @@ harness_config: <harness configuration options>
              tags: sensors
              depends_on: i2c
 
+    The following is an example yaml file with pytest harness_config options, default pytest_root
+    name will be used if pytest_root not specified and the default pytest_timeout is 15 seconds.
+    as an example, test case in test/pytest_sample/ use the default value of these two options.
+
+    ::
+
+        tests:
+          pytest.example:
+            harness: pytest
+            harness_config:
+              pytest_root: [pytest directory name]
+              pytest_timeout: [timeout]
+
 filter: <expression>
     Filter whether the testcase should be run by evaluating an expression
     against an environment containing the following values:

scripts/pylib/twister/harness.py

@@ -1,8 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 import re
+import os
+import subprocess
 from collections import OrderedDict
+import xml.etree.ElementTree as ET
 
 result_re = re.compile(".*(PASS|FAIL|SKIP) - (test_)?(.*)")
+pytest_running_dir = ""
 
 class Harness:
     GCOV_START = "GCOV_COVERAGE_DUMP_START"
@@ -28,6 +32,10 @@ def __init__(self):
         self.recording = []
         self.fieldnames = []
         self.ztest = False
+        self.running_dir = None
+        self.source_dir = None
+        self.pytest_root = 'pytest'
+        self.pytest_timeout = 15
 
     def configure(self, instance):
         config = instance.testcase.harness_config
@@ -41,6 +49,10 @@ def configure(self, instance):
             self.repeat = config.get('repeat', 1)
             self.ordered = config.get('ordered', True)
             self.record = config.get('record', {})
+            self.running_dir = config.get('running_dir', None)
+            self.source_dir = config.get('source_dir', None)
+            self.pytest_root = config.get('pytest_root', 'pytest')
+            self.pytest_timeout = config.get('pytest_timeout', 15)
 
     def process_test(self, line):
 
@@ -121,6 +133,63 @@ def handle(self, line):
 
         self.process_test(line)
 
+class Pytest(Harness):
+
+    def handle(self, line):
+        ''' To keep artifacts of pytest in self.running_dir, pass this directory
+            by "--cmdopt". On pytest end, add a command line option and provide
+            the cmdopt through a fixture function
+			pytest handle output as a whole, it's not necessary to handle it
+			line by line
+        '''
+        global pytest_running_dir
+        if self.running_dir is pytest_running_dir:
+            return
+
+        pytest_running_dir = self.running_dir
+        cmd = [
+			'pytest',
+			'-s',
+			os.path.join(self.source_dir, self.pytest_root),
+			'--cmdopt',
+			self.running_dir,
+			'--junit-xml',
+			os.path.join(self.running_dir, 'report.html'),
+			'-q'
+        ]
+
+        with subprocess.Popen(cmd) as proc:
+            try:
+                proc.wait(self.pytest_timeout)
+                tree = ET.parse(os.path.join(self.running_dir, "report.html"))
+                root = tree.getroot()
+                for child in root:
+                    if child.tag == 'testsuite':
+                        if child.attrib['failures'] != '0':
+                            self.state = "failed"
+                        elif child.attrib['skipped'] != '0':
+                            self.state = "skipped"
+                        elif child.attrib['errors'] != '0':
+                            self.state = "errors"
+                        else:
+                            self.state = "passed"
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                self.state = "failed"
+            except ET.ParseError:
+                self.state = "failed"
+            except IOError:
+                self.state = "failed"
+
+        if self.state == "passed":
+            self.tests[self.id] = "PASS"
+        elif self.state == "skipped":
+            self.tests[self.id] = "SKIP"
+        else:
+            self.tests[self.id] = "FAIL"
+
+        self.process_test(line)
+
 class Test(Harness):
     RUN_PASSED = "PROJECT EXECUTION SUCCESSFUL"
     RUN_FAILED = "PROJECT EXECUTION FAILED"

scripts/pylib/twister/twisterlib.py

@@ -509,6 +509,8 @@ def handle(self):
         harness_import = HarnessImporter(harness_name)
         harness = harness_import.instance
         harness.configure(self.instance)
+        harness.running_dir = self.build_dir
+        harness.source_dir = self.sourcedir
 
         if self.call_make_run:
             command = [self.generator_cmd, "run"]
@@ -789,6 +791,8 @@ def handle(self):
         harness_import = HarnessImporter(harness_name)
         harness = harness_import.instance
         harness.configure(self.instance)
+        harness.running_dir = self.build_dir
+        harness.source_dir = self.sourcedir
         read_pipe, write_pipe = os.pipe()
         start_time = time.time()
 
@@ -1053,6 +1057,9 @@ def handle(self):
         harness_import = HarnessImporter(self.instance.testcase.harness.capitalize())
         harness = harness_import.instance
         harness.configure(self.instance)
+        harness.running_dir = self.build_dir
+        harness.source_dir = self.sourcedir
+
         self.thread = threading.Thread(name=self.name, target=QEMUHandler._thread,
                                        args=(self, self.timeout, self.build_dir,
                                              self.log_fn, self.fifo_fn,
@@ -1757,7 +1764,7 @@ def __lt__(self, other):
     def testcase_runnable(testcase, fixtures):
         can_run = False
         # console harness allows us to run the test and capture data.
-        if testcase.harness in [ 'console', 'ztest']:
+        if testcase.harness in [ 'console', 'ztest', 'pytest']:
             can_run = True
             # if we have a fixture that is also being supplied on the
             # command-line, then we need to run the test, not just build it.

scripts/schemas/twister/testcase-schema.yaml

@@ -62,6 +62,12 @@ mapping:
           "repeat":
             type: int
             required: no
+          "pytest_root":
+            type: str
+            required: no
+          "pytest_timeout":
+            type: int
+            required: no
           "regex":
             type: seq
             required: no
@@ -187,6 +193,12 @@ mapping:
               "repeat":
                 type: int
                 required: no
+              "pytest_root":
+                type: str
+                required: no
+              "pytest_timeout":
+                type: int
+                required: no
               "regex":
                 type: seq
                 required: no

tests/pytest/pytest_sample/CMakeLists.txt

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.13.1)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(pytest_sample)
+
+target_sources(app PRIVATE src/main.c)

tests/pytest/pytest_sample/prj.conf

@@ -0,0 +1,2 @@
+CONFIG_ZTEST=y
+CONFIG_IDLE_STACK_SIZE=4096

tests/pytest/pytest_sample/pytest/conftest.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2020 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--cmdopt'
+    )
+
+@pytest.fixture()
+def cmdopt(request):
+    return request.config.getoption('--cmdopt')

tests/pytest/pytest_sample/pytest/test_ctf.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2020 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import pytest
+
+@pytest.fixture(autouse=True)
+def pytest_cmdopt_handle(cmdopt):
+    ''' handle cmdopt '''
+    print("handle cmdopt:")
+    print(cmdopt)
+    data_path = cmdopt
+    os.environ['data_path'] = str(data_path)
+
+def test_case(cmdopt):
+    ''' test case '''
+    if not os.path.exists(cmdopt):
+        assert 0
+    print("run test cases in:")
+    print(cmdopt)
+
+if __name__ == "__main__":
+    pytest.main()

tests/pytest/pytest_sample/src/main.c

@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <ztest.h>
+void test_pytest(void)
+{
+	TC_PRINT("Hello world\n");
+}
+
+void test_main(void)
+{
+	ztest_test_suite(test_pytest,
+			 ztest_unit_test(test_pytest)
+			 );
+
+	ztest_run_test_suite(test_pytest);
+}

tests/pytest/pytest_sample/testcase.yaml

@@ -0,0 +1,4 @@
+tests:
+  pytest.sample:
+    platform_allow: qemu_x86 qemu_x86_64
+    harness: pytest