scripts: dts: extract pickled EDT generation

Separate the pickled EDT generation from the C-Macro header
generation in gen_defines.py to have a more clear responsibility
of the scripts in the DTS parsing process.

Signed-off-by: Benedikt Schmidt <benedikt.schmidt@embedded-solutions.at>

Author: benediktibk

cmake/modules/dts.cmake

@@ -94,6 +94,8 @@ find_package(Dtc 1.4.6)
 # The directory containing devicetree related scripts.
 set(DT_SCRIPTS                  ${ZEPHYR_BASE}/scripts/dts)
 
+# This parses and collects the DT information
+set(GEN_EDT_SCRIPT              ${DT_SCRIPTS}/gen_edt.py)
 # This generates DT information needed by the C macro APIs,
 # along with a few other things.
 set(GEN_DEFINES_SCRIPT          ${DT_SCRIPTS}/gen_defines.py)
@@ -216,7 +218,7 @@ foreach(dts_root ${DTS_ROOT})
 
   set(vendor_prefixes ${dts_root}/${VENDOR_PREFIXES})
   if(EXISTS ${vendor_prefixes})
-    list(APPEND EXTRA_GEN_DEFINES_ARGS --vendor-prefixes ${vendor_prefixes})
+    list(APPEND EXTRA_GEN_EDT_ARGS --vendor-prefixes ${vendor_prefixes})
   endif()
 endforeach()
 
@@ -266,23 +268,44 @@ toolchain_parse_make_rule(${DTS_DEPS}
 set_property(DIRECTORY APPEND PROPERTY
   CMAKE_CONFIGURE_DEPENDS
   ${DTS_INCLUDE_FILES}
+  ${GEN_EDT_SCRIPT}
   ${GEN_DEFINES_SCRIPT}
   ${GEN_DRIVER_KCONFIG_SCRIPT}
   ${GEN_DTS_CMAKE_SCRIPT}
   )
 
 #
-# Run GEN_DEFINES_SCRIPT.
+# Run GEN_EDT_SCRIPT.
 #
 
 string(REPLACE ";" " " EXTRA_DTC_FLAGS_RAW "${EXTRA_DTC_FLAGS}")
-set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT}
+set(CMD_GEN_EDT ${PYTHON_EXECUTABLE} ${GEN_EDT_SCRIPT}
 --dts ${DTS_POST_CPP}
 --dtc-flags '${EXTRA_DTC_FLAGS_RAW}'
 --bindings-dirs ${DTS_ROOT_BINDINGS}
---header-out ${DEVICETREE_GENERATED_H}.new
 --dts-out ${ZEPHYR_DTS}.new # for debugging and dtc
---edt-pickle-out ${EDT_PICKLE}
+--edt-pickle-out ${EDT_PICKLE}.new
+${EXTRA_GEN_EDT_ARGS}
+)
+
+execute_process(
+  COMMAND ${CMD_GEN_EDT}
+  WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+  COMMAND_ERROR_IS_FATAL ANY
+  )
+zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT)
+zephyr_file_copy(${EDT_PICKLE}.new ${EDT_PICKLE} ONLY_IF_DIFFERENT)
+file(REMOVE ${ZEPHYR_DTS}.new ${EDT_PICKLE}.new)
+message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}")
+message(STATUS "Generated pickled edt: ${EDT_PICKLE}")
+
+#
+# Run GEN_DEFINES_SCRIPT.
+#
+
+set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT}
+--header-out ${DEVICETREE_GENERATED_H}.new
+--edt-pickle ${EDT_PICKLE}
 ${EXTRA_GEN_DEFINES_ARGS}
 )
 
@@ -291,7 +314,6 @@ execute_process(
   WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
   COMMAND_ERROR_IS_FATAL ANY
   )
-zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT)
 zephyr_file_copy(${DEVICETREE_GENERATED_H}.new ${DEVICETREE_GENERATED_H} ONLY_IF_DIFFERENT)
 file(REMOVE ${ZEPHYR_DTS}.new ${DEVICETREE_GENERATED_H}.new)
 message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}")

doc/releases/release-notes-4.0.rst

@@ -133,6 +133,17 @@ Build system and Infrastructure
 
 * Added support for .elf files to the west flash command for jlink, pyocd and linkserver runners.
 
+* Extracted pickled EDT generation from gen_defines.py into gen_edt.py. This moved the following
+  parameters from the cmake variable ``EXTRA_GEN_DEFINES_ARGS`` to ``EXTRA_GEN_EDT_ARGS``:
+
+   * ``--dts``
+   * ``--dtc-flags``
+   * ``--bindings-dirs``
+   * ``--dts-out``
+   * ``--edt-pickle-out``
+   * ``--vendor-prefixes``
+   * ``--edtlib-Werror``
+
 Documentation
 *************
 

scripts/dts/edtlib_logger.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA
+# Copyright (c) 2019 Linaro Limited
+# Copyright (c) 2024 SILA Embedded Solutions GmbH
+
+import logging
+import sys
+
+
+class LogFormatter(logging.Formatter):
+    '''A log formatter that prints the level name in lower case,
+    for compatibility with earlier versions of edtlib.'''
+
+    def __init__(self):
+        super().__init__(fmt='%(levelnamelower)s: %(message)s')
+
+    def format(self, record):
+        record.levelnamelower = record.levelname.lower()
+        return super().format(record)
+
+
+def setup_edtlib_logging() -> None:
+    # The edtlib module emits logs using the standard 'logging' module.
+    # Configure it so that warnings and above are printed to stderr,
+    # using the LogFormatter class defined above to format each message.
+
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(LogFormatter())
+
+    logger = logging.getLogger('edtlib')
+    logger.setLevel(logging.WARNING)
+    logger.addHandler(handler)

scripts/dts/gen_defines.py

@@ -2,16 +2,11 @@
 
 # Copyright (c) 2019 - 2020 Nordic Semiconductor ASA
 # Copyright (c) 2019 Linaro Limited
+# Copyright (c) 2024 SILA Embedded Solutions GmbH
 # SPDX-License-Identifier: BSD-3-Clause
 
-# This script uses edtlib to generate a header file from a devicetree
-# (.dts) file. Information from binding files in YAML format is used
-# as well.
-#
-# Bindings are files that describe devicetree nodes. Devicetree nodes are
-# usually mapped to bindings via their 'compatible = "..."' property.
-#
-# See Zephyr's Devicetree user guide for details.
+# This script uses edtlib to generate a header file from a pickled
+# edt file.
 #
 # Note: Do not access private (_-prefixed) identifiers from edtlib here (and
 # also note that edtlib is not meant to expose the dtlib API directly).
@@ -20,7 +15,6 @@
 
 import argparse
 from collections import defaultdict
-import logging
 import os
 import pathlib
 import pickle
@@ -31,49 +25,23 @@
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'python-devicetree',
                                 'src'))
 
+import edtlib_logger
 from devicetree import edtlib
 
-class LogFormatter(logging.Formatter):
-    '''A log formatter that prints the level name in lower case,
-    for compatibility with earlier versions of edtlib.'''
-
-    def __init__(self):
-        super().__init__(fmt='%(levelnamelower)s: %(message)s')
-
-    def format(self, record):
-        record.levelnamelower = record.levelname.lower()
-        return super().format(record)
 
 def main():
     global header_file
     global flash_area_num
 
     args = parse_args()
 
-    setup_edtlib_logging()
+    edtlib_logger.setup_edtlib_logging()
 
-    vendor_prefixes = {}
-    for prefixes_file in args.vendor_prefixes:
-        vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file))
-
-    try:
-        edt = edtlib.EDT(args.dts, args.bindings_dirs,
-                         # Suppress this warning if it's suppressed in dtc
-                         warn_reg_unit_address_mismatch=
-                             "-Wno-simple_bus_reg" not in args.dtc_flags,
-                         default_prop_types=True,
-                         infer_binding_for_paths=["/zephyr,user"],
-                         werror=args.edtlib_Werror,
-                         vendor_prefixes=vendor_prefixes)
-    except edtlib.EDTError as e:
-        sys.exit(f"devicetree error: {e}")
+    with open(args.edt_pickle, 'rb') as f:
+        edt = pickle.load(f)
 
     flash_area_num = 0
 
-    # Save merged DTS source, as a debugging aid
-    with open(args.dts_out, "w", encoding="utf-8") as f:
-        print(edt.dts_source, file=f)
-
     # Create the generated header.
     with open(args.header_out, "w", encoding="utf-8") as header_file:
         write_top_comment(edt)
@@ -133,22 +101,6 @@ def main():
         write_chosen(edt)
         write_global_macros(edt)
 
-    if args.edt_pickle_out:
-        write_pickled_edt(edt, args.edt_pickle_out)
-
-
-def setup_edtlib_logging() -> None:
-    # The edtlib module emits logs using the standard 'logging' module.
-    # Configure it so that warnings and above are printed to stderr,
-    # using the LogFormatter class defined above to format each message.
-
-    handler = logging.StreamHandler(sys.stderr)
-    handler.setFormatter(LogFormatter())
-
-    logger = logging.getLogger('edtlib')
-    logger.setLevel(logging.WARNING)
-    logger.addHandler(handler)
-
 
 def node_z_path_id(node: edtlib.Node) -> str:
     # Return the node specific bit of the node's path identifier:
@@ -173,27 +125,10 @@ def parse_args() -> argparse.Namespace:
     # Returns parsed command-line arguments
 
     parser = argparse.ArgumentParser(allow_abbrev=False)
-    parser.add_argument("--dts", required=True, help="DTS file")
-    parser.add_argument("--dtc-flags",
-                        help="'dtc' devicetree compiler flags, some of which "
-                             "might be respected here")
-    parser.add_argument("--bindings-dirs", nargs='+', required=True,
-                        help="directory with bindings in YAML format, "
-                        "we allow multiple")
     parser.add_argument("--header-out", required=True,
                         help="path to write header to")
-    parser.add_argument("--dts-out", required=True,
-                        help="path to write merged DTS source code to (e.g. "
-                             "as a debugging aid)")
-    parser.add_argument("--edt-pickle-out",
-                        help="path to write pickled edtlib.EDT object to")
-    parser.add_argument("--vendor-prefixes", action='append', default=[],
-                        help="vendor-prefixes.txt path; used for validation; "
-                             "may be given multiple times")
-    parser.add_argument("--edtlib-Werror", action="store_true",
-                        help="if set, edtlib-specific warnings become errors. "
-                             "(this does not apply to warnings shared "
-                             "with dtc.)")
+    parser.add_argument("--edt-pickle",
+                        help="path to read pickled edtlib.EDT object from")
 
     return parser.parse_args()
 
@@ -1099,21 +1034,6 @@ def quote_str(s: str) -> str:
     return f'"{escape(s)}"'
 
 
-def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None:
-    # Writes the edt object in pickle format to out_file.
-
-    with open(out_file, 'wb') as f:
-        # Pickle protocol version 4 is the default as of Python 3.8
-        # and was introduced in 3.4, so it is both available and
-        # recommended on all versions of Python that Zephyr supports
-        # (at time of writing, Python 3.6 was Zephyr's minimum
-        # version, and 3.8 the most recent CPython release).
-        #
-        # Using a common protocol version here will hopefully avoid
-        # reproducibility issues in different Python installations.
-        pickle.dump(edt, f, protocol=4)
-
-
 def err(s: str) -> NoReturn:
     raise Exception(s)
 

scripts/dts/gen_edt.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA
+# Copyright (c) 2019 Linaro Limited
+# Copyright (c) 2024 SILA Embedded Solutions GmbH
+# SPDX-License-Identifier: Apache-2.0
+
+# This script uses edtlib to generate a pickled edt from a devicetree
+# (.dts) file. Information from binding files in YAML format is used
+# as well.
+#
+# Bindings are files that describe devicetree nodes. Devicetree nodes are
+# usually mapped to bindings via their 'compatible = "..."' property.
+#
+# See Zephyr's Devicetree user guide for details.
+#
+# Note: Do not access private (_-prefixed) identifiers from edtlib here (and
+# also note that edtlib is not meant to expose the dtlib API directly).
+# Instead, think of what API you need, and add it as a public documented API in
+# edtlib. This will keep this script simple.
+
+import argparse
+import os
+import pickle
+import sys
+from typing import NoReturn
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'python-devicetree',
+                                'src'))
+
+import edtlib_logger
+from devicetree import edtlib
+
+
+def main():
+    args = parse_args()
+
+    edtlib_logger.setup_edtlib_logging()
+
+    vendor_prefixes = {}
+    for prefixes_file in args.vendor_prefixes:
+        vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file))
+
+    try:
+        edt = edtlib.EDT(args.dts, args.bindings_dirs,
+                         # Suppress this warning if it's suppressed in dtc
+                         warn_reg_unit_address_mismatch=
+                             "-Wno-simple_bus_reg" not in args.dtc_flags,
+                         default_prop_types=True,
+                         infer_binding_for_paths=["/zephyr,user"],
+                         werror=args.edtlib_Werror,
+                         vendor_prefixes=vendor_prefixes)
+    except edtlib.EDTError as e:
+        sys.exit(f"devicetree error: {e}")
+
+    # Save merged DTS source, as a debugging aid
+    with open(args.dts_out, "w", encoding="utf-8") as f:
+        print(edt.dts_source, file=f)
+
+    write_pickled_edt(edt, args.edt_pickle_out)
+
+
+def parse_args() -> argparse.Namespace:
+    # Returns parsed command-line arguments
+
+    parser = argparse.ArgumentParser(allow_abbrev=False)
+    parser.add_argument("--dts", required=True, help="DTS file")
+    parser.add_argument("--dtc-flags",
+                        help="'dtc' devicetree compiler flags, some of which "
+                             "might be respected here")
+    parser.add_argument("--bindings-dirs", nargs='+', required=True,
+                        help="directory with bindings in YAML format, "
+                        "we allow multiple")
+    parser.add_argument("--dts-out", required=True,
+                        help="path to write merged DTS source code to (e.g. "
+                             "as a debugging aid)")
+    parser.add_argument("--edt-pickle-out",
+                        help="path to write pickled edtlib.EDT object to", required=True)
+    parser.add_argument("--vendor-prefixes", action='append', default=[],
+                        help="vendor-prefixes.txt path; used for validation; "
+                             "may be given multiple times")
+    parser.add_argument("--edtlib-Werror", action="store_true",
+                        help="if set, edtlib-specific warnings become errors. "
+                             "(this does not apply to warnings shared "
+                             "with dtc.)")
+
+    return parser.parse_args()
+
+
+def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None:
+    # Writes the edt object in pickle format to out_file.
+
+    with open(out_file, 'wb') as f:
+        # Pickle protocol version 4 is the default as of Python 3.8
+        # and was introduced in 3.4, so it is both available and
+        # recommended on all versions of Python that Zephyr supports
+        # (at time of writing, Python 3.6 was Zephyr's minimum
+        # version, and 3.10 the most recent CPython release).
+        #
+        # Using a common protocol version here will hopefully avoid
+        # reproducibility issues in different Python installations.
+        pickle.dump(edt, f, protocol=4)
+
+
+def err(s: str) -> NoReturn:
+    raise Exception(s)
+
+
+if __name__ == "__main__":
+    main()