michaelh/zephyr
1
0
Fork 0
Code Releases Activity

scripts: dts: extract pickled EDT generation

Browse source 
Separate the pickled EDT generation from the C-Macro header
generation in gen_defines.py to have a more clear responsibility
of the scripts in the DTS parsing process.

Signed-off-by: Benedikt Schmidt <benedikt.schmidt@embedded-solutions.at>
This commit is contained in:
Benedikt Schmidt 2024-09-09 11:18:56 +02:00 • committed by Mahesh Mahadevan
commit fe3287a9ac
7 changed files with 196 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

34
cmake/modules/dts.cmake
View file

@ -94,6 +94,8 @@ find_package(Dtc 1.4.6)
# The directory containing devicetree related scripts.
set(DT_SCRIPTS ${ZEPHYR_BASE}/scripts/dts)
# This parses and collects the DT information
set(GEN_EDT_SCRIPT ${DT_SCRIPTS}/gen_edt.py)
# This generates DT information needed by the C macro APIs,
# along with a few other things.
set(GEN_DEFINES_SCRIPT ${DT_SCRIPTS}/gen_defines.py)
@ -216,7 +218,7 @@ foreach(dts_root ${DTS_ROOT})
set(vendor_prefixes ${dts_root}/${VENDOR_PREFIXES})
if(EXISTS ${vendor_prefixes})
list(APPEND EXTRA_GEN_DEFINES_ARGS --vendor-prefixes ${vendor_prefixes})
list(APPEND EXTRA_GEN_EDT_ARGS --vendor-prefixes ${vendor_prefixes})
endif()
endforeach()
@ -266,23 +268,44 @@ toolchain_parse_make_rule(${DTS_DEPS}
set_property(DIRECTORY APPEND PROPERTY
CMAKE_CONFIGURE_DEPENDS
${DTS_INCLUDE_FILES}
${GEN_EDT_SCRIPT}
${GEN_DEFINES_SCRIPT}
${GEN_DRIVER_KCONFIG_SCRIPT}
${GEN_DTS_CMAKE_SCRIPT}
)
#
# Run GEN_DEFINES_SCRIPT.
# Run GEN_EDT_SCRIPT.
#
string(REPLACE ";" " " EXTRA_DTC_FLAGS_RAW "${EXTRA_DTC_FLAGS}")
set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT}
set(CMD_GEN_EDT ${PYTHON_EXECUTABLE} ${GEN_EDT_SCRIPT}
--dts ${DTS_POST_CPP}
--dtc-flags '${EXTRA_DTC_FLAGS_RAW}'
--bindings-dirs ${DTS_ROOT_BINDINGS}
--header-out ${DEVICETREE_GENERATED_H}.new
--dts-out ${ZEPHYR_DTS}.new # for debugging and dtc
--edt-pickle-out ${EDT_PICKLE}
--edt-pickle-out ${EDT_PICKLE}.new
${EXTRA_GEN_EDT_ARGS}
)
execute_process(
COMMAND ${CMD_GEN_EDT}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
COMMAND_ERROR_IS_FATAL ANY
)
zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT)
zephyr_file_copy(${EDT_PICKLE}.new ${EDT_PICKLE} ONLY_IF_DIFFERENT)
file(REMOVE ${ZEPHYR_DTS}.new ${EDT_PICKLE}.new)
message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}")
message(STATUS "Generated pickled edt: ${EDT_PICKLE}")
#
# Run GEN_DEFINES_SCRIPT.
#
set(CMD_GEN_DEFINES ${PYTHON_EXECUTABLE} ${GEN_DEFINES_SCRIPT}
--header-out ${DEVICETREE_GENERATED_H}.new
--edt-pickle ${EDT_PICKLE}
${EXTRA_GEN_DEFINES_ARGS}
)
@ -291,7 +314,6 @@ execute_process(
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
COMMAND_ERROR_IS_FATAL ANY
)
zephyr_file_copy(${ZEPHYR_DTS}.new ${ZEPHYR_DTS} ONLY_IF_DIFFERENT)
zephyr_file_copy(${DEVICETREE_GENERATED_H}.new ${DEVICETREE_GENERATED_H} ONLY_IF_DIFFERENT)
file(REMOVE ${ZEPHYR_DTS}.new ${DEVICETREE_GENERATED_H}.new)
message(STATUS "Generated zephyr.dts: ${ZEPHYR_DTS}")

11
doc/releases/release-notes-4.0.rst
View file

@ -133,6 +133,17 @@ Build system and Infrastructure
* Added support for .elf files to the west flash command for jlink, pyocd and linkserver runners.
* Extracted pickled EDT generation from gen_defines.py into gen_edt.py. This moved the following
parameters from the cmake variable ``EXTRA_GEN_DEFINES_ARGS`` to ``EXTRA_GEN_EDT_ARGS``:
* ``--dts``
* ``--dtc-flags``
* ``--bindings-dirs``
* ``--dts-out``
* ``--edt-pickle-out``
* ``--vendor-prefixes``
* ``--edtlib-Werror``
Documentation
*************

33
scripts/dts/edtlib_logger.py Normal file
View file

@ -0,0 +1,33 @@
#!/usr/bin/env python3
# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA
# Copyright (c) 2019 Linaro Limited
# Copyright (c) 2024 SILA Embedded Solutions GmbH
import logging
import sys
class LogFormatter(logging.Formatter):
'''A log formatter that prints the level name in lower case,
for compatibility with earlier versions of edtlib.'''
def __init__(self):
super().__init__(fmt='%(levelnamelower)s: %(message)s')
def format(self, record):
record.levelnamelower = record.levelname.lower()
return super().format(record)
def setup_edtlib_logging() -> None:
# The edtlib module emits logs using the standard 'logging' module.
# Configure it so that warnings and above are printed to stderr,
# using the LogFormatter class defined above to format each message.
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(LogFormatter())
logger = logging.getLogger('edtlib')
logger.setLevel(logging.WARNING)
logger.addHandler(handler)

98
scripts/dts/gen_defines.py
View file

@ -2,16 +2,11 @@
# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA
# Copyright (c) 2019 Linaro Limited
# Copyright (c) 2024 SILA Embedded Solutions GmbH
# SPDX-License-Identifier: BSD-3-Clause
# This script uses edtlib to generate a header file from a devicetree
# (.dts) file. Information from binding files in YAML format is used
# as well.
#
# Bindings are files that describe devicetree nodes. Devicetree nodes are
# usually mapped to bindings via their 'compatible = "..."' property.
#
# See Zephyr's Devicetree user guide for details.
# This script uses edtlib to generate a header file from a pickled
# edt file.
#
# Note: Do not access private (_-prefixed) identifiers from edtlib here (and
# also note that edtlib is not meant to expose the dtlib API directly).
@ -20,7 +15,6 @@
import argparse
from collections import defaultdict
import logging
import os
import pathlib
import pickle
@ -31,18 +25,9 @@ from typing import Iterable, NoReturn, Optional
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'python-devicetree',
'src'))
import edtlib_logger
from devicetree import edtlib
class LogFormatter(logging.Formatter):
'''A log formatter that prints the level name in lower case,
for compatibility with earlier versions of edtlib.'''
def __init__(self):
super().__init__(fmt='%(levelnamelower)s: %(message)s')
def format(self, record):
record.levelnamelower = record.levelname.lower()
return super().format(record)
def main():
global header_file
@ -50,30 +35,13 @@ def main():
args = parse_args()
setup_edtlib_logging()
edtlib_logger.setup_edtlib_logging()
vendor_prefixes = {}
for prefixes_file in args.vendor_prefixes:
vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file))
try:
edt = edtlib.EDT(args.dts, args.bindings_dirs,
# Suppress this warning if it's suppressed in dtc
warn_reg_unit_address_mismatch=
"-Wno-simple_bus_reg" not in args.dtc_flags,
default_prop_types=True,
infer_binding_for_paths=["/zephyr,user"],
werror=args.edtlib_Werror,
vendor_prefixes=vendor_prefixes)
except edtlib.EDTError as e:
sys.exit(f"devicetree error: {e}")
with open(args.edt_pickle, 'rb') as f:
edt = pickle.load(f)
flash_area_num = 0
# Save merged DTS source, as a debugging aid
with open(args.dts_out, "w", encoding="utf-8") as f:
print(edt.dts_source, file=f)
# Create the generated header.
with open(args.header_out, "w", encoding="utf-8") as header_file:
write_top_comment(edt)
@ -133,22 +101,6 @@ def main():
write_chosen(edt)
write_global_macros(edt)
if args.edt_pickle_out:
write_pickled_edt(edt, args.edt_pickle_out)
def setup_edtlib_logging() -> None:
# The edtlib module emits logs using the standard 'logging' module.
# Configure it so that warnings and above are printed to stderr,
# using the LogFormatter class defined above to format each message.
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(LogFormatter())
logger = logging.getLogger('edtlib')
logger.setLevel(logging.WARNING)
logger.addHandler(handler)
def node_z_path_id(node: edtlib.Node) -> str:
# Return the node specific bit of the node's path identifier:
@ -173,27 +125,10 @@ def parse_args() -> argparse.Namespace:
# Returns parsed command-line arguments
parser = argparse.ArgumentParser(allow_abbrev=False)
parser.add_argument("--dts", required=True, help="DTS file")
parser.add_argument("--dtc-flags",
help="'dtc' devicetree compiler flags, some of which "
"might be respected here")
parser.add_argument("--bindings-dirs", nargs='+', required=True,
help="directory with bindings in YAML format, "
"we allow multiple")
parser.add_argument("--header-out", required=True,
help="path to write header to")
parser.add_argument("--dts-out", required=True,
help="path to write merged DTS source code to (e.g. "
"as a debugging aid)")
parser.add_argument("--edt-pickle-out",
help="path to write pickled edtlib.EDT object to")
parser.add_argument("--vendor-prefixes", action='append', default=[],
help="vendor-prefixes.txt path; used for validation; "
"may be given multiple times")
parser.add_argument("--edtlib-Werror", action="store_true",
help="if set, edtlib-specific warnings become errors. "
"(this does not apply to warnings shared "
"with dtc.)")
parser.add_argument("--edt-pickle",
help="path to read pickled edtlib.EDT object from")
return parser.parse_args()
@ -1099,21 +1034,6 @@ def quote_str(s: str) -> str:
return f'"{escape(s)}"'
def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None:
# Writes the edt object in pickle format to out_file.
with open(out_file, 'wb') as f:
# Pickle protocol version 4 is the default as of Python 3.8
# and was introduced in 3.4, so it is both available and
# recommended on all versions of Python that Zephyr supports
# (at time of writing, Python 3.6 was Zephyr's minimum
# version, and 3.8 the most recent CPython release).
#
# Using a common protocol version here will hopefully avoid
# reproducibility issues in different Python installations.
pickle.dump(edt, f, protocol=4)
def err(s: str) -> NoReturn:
raise Exception(s)

110
scripts/dts/gen_edt.py Executable file
View file

@ -0,0 +1,110 @@
#!/usr/bin/env python3
# Copyright (c) 2019 - 2020 Nordic Semiconductor ASA
# Copyright (c) 2019 Linaro Limited
# Copyright (c) 2024 SILA Embedded Solutions GmbH
# SPDX-License-Identifier: Apache-2.0
# This script uses edtlib to generate a pickled edt from a devicetree
# (.dts) file. Information from binding files in YAML format is used
# as well.
#
# Bindings are files that describe devicetree nodes. Devicetree nodes are
# usually mapped to bindings via their 'compatible = "..."' property.
#
# See Zephyr's Devicetree user guide for details.
#
# Note: Do not access private (_-prefixed) identifiers from edtlib here (and
# also note that edtlib is not meant to expose the dtlib API directly).
# Instead, think of what API you need, and add it as a public documented API in
# edtlib. This will keep this script simple.
import argparse
import os
import pickle
import sys
from typing import NoReturn
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'python-devicetree',
'src'))
import edtlib_logger
from devicetree import edtlib
def main():
args = parse_args()
edtlib_logger.setup_edtlib_logging()
vendor_prefixes = {}
for prefixes_file in args.vendor_prefixes:
vendor_prefixes.update(edtlib.load_vendor_prefixes_txt(prefixes_file))
try:
edt = edtlib.EDT(args.dts, args.bindings_dirs,
# Suppress this warning if it's suppressed in dtc
warn_reg_unit_address_mismatch=
"-Wno-simple_bus_reg" not in args.dtc_flags,
default_prop_types=True,
infer_binding_for_paths=["/zephyr,user"],
werror=args.edtlib_Werror,
vendor_prefixes=vendor_prefixes)
except edtlib.EDTError as e:
sys.exit(f"devicetree error: {e}")
# Save merged DTS source, as a debugging aid
with open(args.dts_out, "w", encoding="utf-8") as f:
print(edt.dts_source, file=f)
write_pickled_edt(edt, args.edt_pickle_out)
def parse_args() -> argparse.Namespace:
# Returns parsed command-line arguments
parser = argparse.ArgumentParser(allow_abbrev=False)
parser.add_argument("--dts", required=True, help="DTS file")
parser.add_argument("--dtc-flags",
help="'dtc' devicetree compiler flags, some of which "
"might be respected here")
parser.add_argument("--bindings-dirs", nargs='+', required=True,
help="directory with bindings in YAML format, "
"we allow multiple")
parser.add_argument("--dts-out", required=True,
help="path to write merged DTS source code to (e.g. "
"as a debugging aid)")
parser.add_argument("--edt-pickle-out",
help="path to write pickled edtlib.EDT object to", required=True)
parser.add_argument("--vendor-prefixes", action='append', default=[],
help="vendor-prefixes.txt path; used for validation; "
"may be given multiple times")
parser.add_argument("--edtlib-Werror", action="store_true",
help="if set, edtlib-specific warnings become errors. "
"(this does not apply to warnings shared "
"with dtc.)")
return parser.parse_args()
def write_pickled_edt(edt: edtlib.EDT, out_file: str) -> None:
# Writes the edt object in pickle format to out_file.
with open(out_file, 'wb') as f:
# Pickle protocol version 4 is the default as of Python 3.8
# and was introduced in 3.4, so it is both available and
# recommended on all versions of Python that Zephyr supports
# (at time of writing, Python 3.6 was Zephyr's minimum
# version, and 3.10 the most recent CPython release).
#
# Using a common protocol version here will hopefully avoid
# reproducibility issues in different Python installations.
pickle.dump(edt, f, protocol=4)
def err(s: str) -> NoReturn:
raise Exception(s)
if __name__ == "__main__":
main()

6
scripts/pylib/twister/twisterlib/runner.py
View file

@ -330,10 +330,10 @@ class CMake:
if not self.options.disable_warnings_as_errors:
warnings_as_errors = 'y'
gen_defines_args = "--edtlib-Werror"
gen_edt_args = "--edtlib-Werror"
else:
warnings_as_errors = 'n'
gen_defines_args = ""
gen_edt_args = ""
warning_command = 'CONFIG_COMPILER_WARNINGS_AS_ERRORS'
if self.instance.sysbuild:
@ -345,7 +345,7 @@ class CMake:
f'-DTC_RUNID={self.instance.run_id}',
f'-DTC_NAME={self.instance.testsuite.name}',
f'-D{warning_command}={warnings_as_errors}',
f'-DEXTRA_GEN_DEFINES_ARGS={gen_defines_args}',
f'-DEXTRA_GEN_EDT_ARGS={gen_edt_args}',
f'-G{self.env.generator}'
]

4
scripts/tests/twister/test_runner.py
View file

@ -371,7 +371,7 @@ TESTDATA_2_2 = [
[os.path.join('dummy', 'cmake'),
'-B' + os.path.join('build', 'dir'), '-DTC_RUNID=1', '-DTC_NAME=testcase',
'-DSB_CONFIG_COMPILER_WARNINGS_AS_ERRORS=y',
'-DEXTRA_GEN_DEFINES_ARGS=--edtlib-Werror', '-Gdummy_generator',
'-DEXTRA_GEN_EDT_ARGS=--edtlib-Werror', '-Gdummy_generator',
'-S' + os.path.join('source', 'dir'),
'arg1', 'arg2',
'-DBOARD=<platform name>',
@ -385,7 +385,7 @@ TESTDATA_2_2 = [
[os.path.join('dummy', 'cmake'),
'-B' + os.path.join('build', 'dir'), '-DTC_RUNID=1', '-DTC_NAME=testcase',
'-DSB_CONFIG_COMPILER_WARNINGS_AS_ERRORS=n',
'-DEXTRA_GEN_DEFINES_ARGS=', '-Gdummy_generator',
'-DEXTRA_GEN_EDT_ARGS=', '-Gdummy_generator',
'-Szephyr_base/share/sysbuild',
'-DAPP_DIR=' + os.path.join('source', 'dir'),
'arg1', 'arg2',