From 62d57414762d111b6af65eeeb7a8325d1ba94de4 Mon Sep 17 00:00:00 2001 From: Ulf Magnusson Date: Mon, 17 Dec 2018 20:09:47 +0100 Subject: [PATCH] dts: Add new DTS/binding parser Add a new DTS/binding parser to scripts/dts/ for generating generated_dts_board.conf and generated_dts_board_unfixed.h. The old code is kept to generate some deprecated defines, using the --deprecated-only flag. It will be removed later. The new parser is implemented in three files in scripts/dts/: dtlib.py: A low-level .dts parsing library. This is similar to devicetree.py in the old code, but is a general robust DTS parser that doesn't rely on preprocessing. edtlib.py (e for extended): A library built on top of dtlib.py that brings together data from DTS files and bindings and creates Device instances with all the data for a device. gen_defines.py: A script that uses edtlib.py to generate generated_dts_board.conf and generated_dts_board_unfixed.h. Corresponds to extract_dts_includes.py and the files in extract/ in the old code. testdtlib.py: Test suite for dtlib.py. Can be run directly as a script. testedtlib.py (uses test.dts and test-bindings/): Test suite for edtlib.py. Can be run directly as a script. The test suites will be run automatically in CI. The new code turns some things that were warnings (or not checked) in the old code into errors, like missing properties that are specified with 'category: required' in the binding for the node. The code includes lots of documentation and tries to give helpful error messages instead of Python errors. Co-authored-by: Kumar Gala Signed-off-by: Ulf Magnusson --- CODEOWNERS | 1 + cmake/dts.cmake | 32 +- include/generated_dts_board.h | 2 + scripts/dts/dtlib.py | 1689 ++++++++++++++ scripts/dts/edtlib.py | 1639 +++++++++++++ scripts/dts/gen_defines.py | 619 +++++ scripts/dts/test-bindings/child.yaml | 14 + scripts/dts/test-bindings/clock-1-cell.yaml | 14 + scripts/dts/test-bindings/clock-2-cell.yaml | 15 + scripts/dts/test-bindings/fixed-clock.yaml | 14 + scripts/dts/test-bindings/gpio-1-cell.yaml | 14 + scripts/dts/test-bindings/gpio-2-cell.yaml | 15 + scripts/dts/test-bindings/grandchild.yaml | 15 + .../dts/test-bindings/interrupt-1-cell.yaml | 14 + .../dts/test-bindings/interrupt-2-cell.yaml | 15 + .../dts/test-bindings/interrupt-3-cell.yaml | 16 + scripts/dts/test-bindings/parent.yaml | 19 + scripts/dts/test-bindings/props.yaml | 26 + scripts/dts/test-bindings/pwm-0-cell.yaml | 11 + scripts/dts/test-bindings/pwm-1-cell.yaml | 14 + .../dts/test-bindings/sub-node-parent.yaml | 21 + scripts/dts/test.dts | 299 +++ scripts/dts/testdtlib.py | 2025 +++++++++++++++++ scripts/dts/testedtlib.py | 123 + 24 files changed, 6662 insertions(+), 4 deletions(-) create mode 100644 scripts/dts/dtlib.py create mode 100644 scripts/dts/edtlib.py create mode 100755 scripts/dts/gen_defines.py create mode 100644 scripts/dts/test-bindings/child.yaml create mode 100644 scripts/dts/test-bindings/clock-1-cell.yaml create mode 100644 scripts/dts/test-bindings/clock-2-cell.yaml create mode 100644 scripts/dts/test-bindings/fixed-clock.yaml create mode 100644 scripts/dts/test-bindings/gpio-1-cell.yaml create mode 100644 scripts/dts/test-bindings/gpio-2-cell.yaml create mode 100644 scripts/dts/test-bindings/grandchild.yaml create mode 100644 scripts/dts/test-bindings/interrupt-1-cell.yaml create mode 100644 scripts/dts/test-bindings/interrupt-2-cell.yaml create mode 100644 scripts/dts/test-bindings/interrupt-3-cell.yaml create mode 100644 scripts/dts/test-bindings/parent.yaml create mode 100644 scripts/dts/test-bindings/props.yaml create mode 100644 scripts/dts/test-bindings/pwm-0-cell.yaml create mode 100644 scripts/dts/test-bindings/pwm-1-cell.yaml create mode 100644 scripts/dts/test-bindings/sub-node-parent.yaml create mode 100644 scripts/dts/test.dts create mode 100755 scripts/dts/testdtlib.py create mode 100755 scripts/dts/testedtlib.py diff --git a/CODEOWNERS b/CODEOWNERS index f89cc36d6d3..c3774cc98ef 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -311,6 +311,7 @@ /scripts/elf_helper.py @andrewboie /scripts/sanity_chk/expr_parser.py @nashif /scripts/gen_app_partitions.py @andrewboie +/scripts/dts/ @ulfalizer @galak /arch/x86/gen_gdt.py @andrewboie /arch/x86/gen_idt.py @andrewboie /scripts/gen_kobject_list.py @andrewboie diff --git a/cmake/dts.cmake b/cmake/dts.cmake index eab376283a0..9704f124e04 100644 --- a/cmake/dts.cmake +++ b/cmake/dts.cmake @@ -156,16 +156,39 @@ if(SUPPORTS_DTS) message(FATAL_ERROR "command failed with return code: ${ret}") endif() + # + # Run gen_defines.py to create a .conf file and a header file + # + + set(CMD_NEW_EXTRACT ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/dts/gen_defines.py + --dts ${BOARD}.dts.pre.tmp + --bindings-dir ${DTS_ROOT_BINDINGS} + --conf-out ${GENERATED_DTS_BOARD_CONF} + --header-out ${GENERATED_DTS_BOARD_UNFIXED_H} + ) + + execute_process( + COMMAND ${CMD_NEW_EXTRACT} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + RESULT_VARIABLE ret + ) + if(NOT "${ret}" STREQUAL "0") + message(FATAL_ERROR "new extractor failed with return code: ${ret}") + endif() + + # + # Run extract_dts_includes.py (the older DT/binding parser) to generate some + # legacy identifiers (via --deprecated-only). This will go away later. + # + set(CMD_EXTRACT_DTS_INCLUDES ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/dts/extract_dts_includes.py + --deprecated-only --dts ${BOARD}.dts_compiled --yaml ${DTS_ROOT_BINDINGS} - --keyvalue ${GENERATED_DTS_BOARD_CONF} - --include ${GENERATED_DTS_BOARD_UNFIXED_H} + --include ${GENERATED_DTS_BOARD_UNFIXED_H}.deprecated --old-alias-names ) - # Run extract_dts_includes.py to create a .conf and a header file that can be - # included into the CMake namespace execute_process( COMMAND ${CMD_EXTRACT_DTS_INCLUDES} WORKING_DIRECTORY ${PROJECT_BINARY_DIR} @@ -179,4 +202,5 @@ if(SUPPORTS_DTS) else() file(WRITE ${GENERATED_DTS_BOARD_UNFIXED_H} "/* WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY! */") + file(WRITE ${GENERATED_DTS_BOARD_UNFIXED_H}.deprecated "/* WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY! */") endif(SUPPORTS_DTS) diff --git a/include/generated_dts_board.h b/include/generated_dts_board.h index ea0615c872f..a48f3f858da 100644 --- a/include/generated_dts_board.h +++ b/include/generated_dts_board.h @@ -13,6 +13,8 @@ #include +#include + /* The following definitions fixup the generated include */ #include diff --git a/scripts/dts/dtlib.py b/scripts/dts/dtlib.py new file mode 100644 index 00000000000..373585e671a --- /dev/null +++ b/scripts/dts/dtlib.py @@ -0,0 +1,1689 @@ +# Copyright (c) 2019, Nordic Semiconductor +# SPDX-License-Identifier: BSD-3-Clause + +# Tip: You can view just the documentation with 'pydoc3 dtlib' + +""" +A library for extracting information from .dts (Device Tree) files. See the +documentation for the DT and Node classes for more information. + +The top-level entry point of the library is the DT class. DT.__init__() takes a +.dts file to parse and a list of directories to search for any /include/d +files. +""" + +import collections +import errno +import os +import re +import sys +import textwrap + +# NOTE: testdtlib.py is the test suite for this library. It can be run directly. + + +class DT: + """ + Represents a device tree parsed from a .dts file (or from many files, if + the .dts file /include/s other files). Creating many instances of this + class is fine. The library has no global state. + + These attributes are available on DT instances: + + root: + A Node instance representing the root (/) node. + + alias2node: + A dictionary that maps maps alias strings (from /aliases) to Node + instances + + label2node: + A dictionary that maps each node label (a string) to the Node instance + for the node. + + label2prop: + A dictionary that maps each property label (a string) to a Property + instance. + + label2prop_offset: + A dictionary that maps each label (a string) within a property value + (e.g., 'x = label_1: < 1 label2: 2 >;') to a (prop, offset) tuple, where + 'prop' is a Property instance and 'offset' the byte offset (0 for label_1 + and 4 for label_2 in the example). + + phandle2node: + A dictionary that maps each phandle (a number) to a Node instance. + + memreserves: + A list of (labels, address, length) tuples for the /memreserve/s in the + .dts file, in the same order as they appear in the file. + + 'labels' is a possibly empty set with all labels preceding the memreserve + (e.g., 'label1: label2: /memreserve/ ...'). 'address' and 'length' are + numbers. + + filename: + The filename passed to the DT constructor. + """ + + # + # Public interface + # + + def __init__(self, filename, include_path=()): + """ + Parses a DTS file to create a DT instance. Raises OSError if 'filename' + can't be opened, and DTError for any parse errors. + + filename: + Path to the .dts file to parse. + + include_path: + An iterable (e.g. list or tuple) containing paths to search for + /include/d and /incbin/'d files. By default, files are only looked up + relative to the .dts file that contains the /include/ or /incbin/. + """ + self.filename = filename + self._include_path = include_path + + with open(filename, encoding="utf-8") as f: + self._file_contents = f.read() + + self._tok_i = self._tok_end_i = 0 + self._filestack = [] + + self.alias2node = {} + + self._lexer_state = _DEFAULT + self._saved_token = None + + self._lineno = 1 + + self._is_parsing = True + + self._parse_dt() + + self._register_phandles() + self._fixup_props() + self._register_aliases() + self._remove_unreferenced() + self._register_labels() + + self._is_parsing = False + + def get_node(self, path): + """ + Returns the Node instance for the node with path or alias 'path' (a + string). Raises DTError if the path or alias doesn't exist. + + For example, both dt.get_node("/foo/bar") and dt.get_node("bar-alias") + will return the 'bar' node below: + + /dts-v1/; + + / { + foo { + bar_label: bar { + baz { + }; + }; + }; + + aliases { + bar-alias = &bar-label; + }; + }; + + Fetching subnodes via aliases is supported: + dt.get_node("bar-alias/baz") returns the 'baz' node. + """ + if path.startswith("/"): + cur = self.root + component_i = 0 + rest = path + else: + # Strip the first component from 'path' and store it in 'alias'. + # Use a separate 'rest' variable rather than directly modifying + # 'path' so that all of 'path' still shows up in error messages. + alias, _, rest = path.partition("/") + if alias not in self.alias2node: + raise DTError("node path does not start with '/'" + if self._is_parsing else + "no alias '{}' found -- did you forget the " + "leading '/' in the node path?".format(alias)) + cur = self.alias2node[alias] + component_i = 1 + + for component in rest.split("/"): + # Collapse multiple / in a row, and allow a / at the end + if not component: + continue + + component_i += 1 + + if component not in cur.nodes: + raise DTError("component {} ({}) in path {} does not exist" + .format(component_i, repr(component), + repr(path))) + + cur = cur.nodes[component] + + return cur + + def has_node(self, path): + """ + Returns True if the path or alias 'path' exists. See Node.get_node(). + """ + try: + self.get_node(path) + return True + except DTError: + return False + + def node_iter(self): + """ + Returns a generator for iterating over all nodes in the device tree. + + For example, this will print the name of each node that has a property + called 'foo': + + for node in dt.node_iter(): + if "foo" in node.props: + print(node.name) + """ + yield from self.root.node_iter() + + def __str__(self): + """ + Returns a DTS representation of the device tree. Called automatically + if the DT instance is print()ed. + """ + s = "/dts-v1/;\n\n" + + if self.memreserves: + for labels, address, offset in self.memreserves: + # List the labels in a consistent order to help with testing + for label in labels: + s += label + ": " + s += "/memreserve/ {:#018x} {:#018x};\n" \ + .format(address, offset) + s += "\n" + + return s + str(self.root) + + def __repr__(self): + """ + Returns some information about the DT instance. Called automatically if + the DT instance is evaluated. + """ + return "DT(filename='{}', include_path={})" \ + .format(self.filename, self._include_path) + + # + # Parsing + # + + def _parse_dt(self): + # Top-level parsing loop + + self._parse_header() + self._parse_memreserves() + + self.root = None + + while True: + tok = self._next_token() + + if tok.val == "/": + # '/ { ... };', the root node + if not self.root: + self.root = Node(name="/", parent=None, dt=self) + self._parse_node(self.root) + + elif tok.id in (_T_LABEL, _T_REF): + # '&foo { ... };' or 'label: &foo { ... };'. The C tools only + # support a single label here too. + + if tok.id is _T_LABEL: + label = tok.val + tok = self._next_token() + if tok.id is not _T_REF: + self._parse_error("expected label reference (&foo)") + else: + label = None + + try: + node = self._ref2node(tok.val) + except DTError as e: + self._parse_error(e) + node = self._parse_node(node) + + if label: + _append_no_dup(node.labels, label) + + elif tok.id is _T_DEL_NODE: + try: + self._del_node(self._next_ref2node()) + except DTError as e: + self._parse_error(e) + self._expect_token(";") + + elif tok.id is _T_OMIT_IF_NO_REF: + try: + self._next_ref2node()._omit_if_no_ref = True + except DTError as e: + self._parse_error(e) + self._expect_token(";") + + elif tok.id is _T_EOF: + if not self.root: + self._parse_error("no root node defined") + return + + else: + self._parse_error("expected '/' or label reference (&foo)") + + def _parse_header(self): + # Parses /dts-v1/ (expected) and /plugin/ (unsupported) at the start of + # files. There may be multiple /dts-v1/ at the start of a file. + + has_dts_v1 = False + + while self._peek_token().id is _T_DTS_V1: + has_dts_v1 = True + self._next_token() + self._expect_token(";") + # /plugin/ always comes after /dts-v1/ + if self._peek_token().id is _T_PLUGIN: + self._parse_error("/plugin/ is not supported") + + if not has_dts_v1: + self._parse_error("expected '/dts-v1/;' at start of file") + + def _parse_memreserves(self): + # Parses /memreserve/, which appears after /dts-v1/ + + self.memreserves = [] + while True: + # Labels before /memreserve/ + labels = [] + while self._peek_token().id is _T_LABEL: + _append_no_dup(labels, self._next_token().val) + + if self._peek_token().id is _T_MEMRESERVE: + self._next_token() + self.memreserves.append( + (labels, self._eval_prim(), self._eval_prim())) + self._expect_token(";") + else: + if labels: + self._parse_error("expected /memreserve/ after labels at " + "beginning of file") + + return + + def _parse_node(self, node): + # Parses the '{ ... };' part of 'node-name { ... };'. Returns the new + # Node. + + self._expect_token("{") + while True: + labels, omit_if_no_ref = self._parse_propnode_labels() + tok = self._next_token() + + if tok.id is _T_PROPNODENAME: + if self._peek_token().val == "{": + # ' { ...', expect node + + if tok.val.count("@") > 1: + self._parse_error("multiple '@' in node name") + + # Fetch the existing node if it already exists. This + # happens when overriding nodes. + child = node.nodes.get(tok.val) or \ + Node(name=tok.val, parent=node, dt=self) + + for label in labels: + _append_no_dup(child.labels, label) + + if omit_if_no_ref: + child._omit_if_no_ref = True + + node.nodes[child.name] = child + self._parse_node(child) + + else: + # Not ' { ...', expect property assignment + + if omit_if_no_ref: + self._parse_error( + "/omit-if-no-ref/ can only be used on nodes") + + prop = self._node_prop(node, tok.val) + + if self._check_token("="): + self._parse_assignment(prop) + elif not self._check_token(";"): + # ';' is for an empty property, like 'foo;' + self._parse_error("expected '{', '=', or ';'") + + for label in labels: + _append_no_dup(prop.labels, label) + + elif tok.id is _T_DEL_NODE: + tok2 = self._next_token() + if tok2.id is not _T_PROPNODENAME: + self._parse_error("expected node name") + if tok2.val in node.nodes: + self._del_node(node.nodes[tok2.val]) + self._expect_token(";") + + elif tok.id is _T_DEL_PROP: + tok2 = self._next_token() + if tok2.id is not _T_PROPNODENAME: + self._parse_error("expected property name") + node.props.pop(tok2.val, None) + self._expect_token(";") + + elif tok.val == "}": + self._expect_token(";") + return node + + else: + self._parse_error("expected node name, property name, or '}'") + + def _parse_propnode_labels(self): + # _parse_node() helpers for parsing labels and /omit-if-no-ref/s before + # nodes and properties. Returns a (