From ccd15df510162687c904dad90c8ab165d0070c95 Mon Sep 17 00:00:00 2001 From: Corey Wharton Date: Sat, 29 Feb 2020 14:51:42 -0800 Subject: [PATCH] scripts: Dynamically add driver subsystems to subsystems list This change extends the parse_syscalls.py script to scan for a __subsystem sentinal added to driver api declarations. It thens generates a list that is passed into gen_kobject_list.py to extend the subsystems list. This allows subsystems to be declared in the code instead of a separate python list and provides a mechanism for defining out-of-tree subsystems. Signed-off-by: Corey Wharton --- CMakeLists.txt | 18 +++++++-- cmake/kobj.cmake | 5 ++- include/toolchain/common.h | 5 +++ scripts/gen_kobject_list.py | 17 +++++++- scripts/parse_syscalls.py | 78 ++++++++++++++++++++++++------------- 5 files changed, 90 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c786770bee4..82407c7284c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -494,6 +494,7 @@ endif() set(syscall_list_h ${CMAKE_CURRENT_BINARY_DIR}/include/generated/syscall_list.h) set(syscalls_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls.json) +set(subsys_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/subsystems.json) # The syscalls subdirs txt file is constructed by python containing a list of folders to use for # dependency handling, including empty folders. @@ -585,12 +586,14 @@ endforeach() add_custom_command( OUTPUT ${syscalls_json} + ${subsys_json} COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/parse_syscalls.py --include ${ZEPHYR_BASE}/include # Read files from this dir ${parse_syscalls_include_args} # Read files from these dirs also - --json-file ${syscalls_json} # Write this file + --json-file ${syscalls_json} # Write this file + --subsystem-file ${subsys_json} # Write subsystem list to this file DEPENDS ${syscalls_subdirs_trigger} ${PARSE_SYSCALLS_HEADER_DEPENDS} ) @@ -617,6 +620,9 @@ add_custom_command(OUTPUT include/generated/syscall_dispatch.c ${syscall_list_h} DEPENDS ${syscalls_json} ) +# This is passed into all calls to the gen_kobject_list.py script. +set(gen_kobject_list_include_args --include ${subsys_json}) + set(DRV_VALIDATION ${PROJECT_BINARY_DIR}/include/generated/driver-validation.h) add_custom_command( OUTPUT ${DRV_VALIDATION} @@ -624,8 +630,11 @@ add_custom_command( ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/gen_kobject_list.py --validation-output ${DRV_VALIDATION} + ${gen_kobject_list_include_args} $<$:--verbose> - DEPENDS ${ZEPHYR_BASE}/scripts/gen_kobject_list.py + DEPENDS + ${ZEPHYR_BASE}/scripts/gen_kobject_list.py + ${subsys_json} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) add_custom_target(${DRIVER_VALIDATION_H_TARGET} DEPENDS ${DRV_VALIDATION}) @@ -941,8 +950,11 @@ if(CONFIG_USERSPACE) ${GEN_KOBJ_LIST} --kernel $ --gperf-output ${OBJ_LIST} + ${gen_kobject_list_include_args} $<$:--verbose> - DEPENDS ${ZEPHYR_PREBUILT_EXECUTABLE} + DEPENDS + ${ZEPHYR_PREBUILT_EXECUTABLE} + ${subsys_json} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) add_custom_target(obj_list DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${OBJ_LIST}) diff --git a/cmake/kobj.cmake b/cmake/kobj.cmake index bd16d3538d9..f54e148a60d 100644 --- a/cmake/kobj.cmake +++ b/cmake/kobj.cmake @@ -21,8 +21,11 @@ function(gen_kobj gen_dir_out) --kobj-types-output ${KOBJ_TYPES} --kobj-otype-output ${KOBJ_OTYPE} --kobj-size-output ${KOBJ_SIZE} + ${gen_kobject_list_include_args} $<$:--verbose> - DEPENDS $ENV{ZEPHYR_BASE}/scripts/gen_kobject_list.py + DEPENDS + $ENV{ZEPHYR_BASE}/scripts/gen_kobject_list.py + ${subsys_json} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) add_custom_target(${KOBJ_TYPES_H_TARGET} DEPENDS ${KOBJ_TYPES} ${KOBJ_OTYPE}) diff --git a/include/toolchain/common.h b/include/toolchain/common.h index 6f48245fa7a..331ed433be0 100644 --- a/include/toolchain/common.h +++ b/include/toolchain/common.h @@ -132,6 +132,11 @@ #define __syscall #endif /* #ifndef ZTEST_UNITTEST */ +/* Used as a sentinel by parse_syscalls.py to identify what API structs + * define driver subsystems. + */ +#define __subsystem + #ifndef BUILD_ASSERT /* compile-time assertion that makes the build fail */ #define BUILD_ASSERT(EXPR) \ diff --git a/scripts/gen_kobject_list.py b/scripts/gen_kobject_list.py index 596102627c7..f0f98115887 100755 --- a/scripts/gen_kobject_list.py +++ b/scripts/gen_kobject_list.py @@ -56,6 +56,7 @@ import argparse import math import os import struct +import json from elf_helper import ElfHelper, kobject_to_enum from collections import OrderedDict @@ -90,8 +91,6 @@ kobjects = OrderedDict([ ("k_futex", (None, True)) ]) - - subsystems = [ "adc_driver_api", "aio_cmp_driver_api", @@ -331,6 +330,11 @@ def write_kobj_size_output(fp): fp.write("#endif\n") +def parse_subsystems_list_file(path): + with open(path, "r") as fp: + subsys_list = json.load(fp) + subsystems.extend(subsys_list) + def parse_args(): global args @@ -355,6 +359,11 @@ def parse_args(): parser.add_argument( "-Z", "--kobj-size-output", required=False, help="Output case statements for obj_size_get()") + parser.add_argument("-i", "--include-subsystem-list", required=False, action='append', + help='''Specifies a file with a JSON encoded list of subsystem names to append to + the driver subsystems list. Can be specified multiple times: + -i file1 -i file2 ...''') + parser.add_argument("-v", "--verbose", action="store_true", help="Print extra debugging information") args = parser.parse_args() @@ -365,6 +374,10 @@ def parse_args(): def main(): parse_args() + if args.include_subsystem_list is not None: + for list_file in args.include_subsystem_list: + parse_subsystems_list_file(list_file) + if args.gperf_output: assert args.kernel, "--kernel ELF required for --gperf-output" eh = ElfHelper(args.kernel, args.verbose, kobjects, subsystems) diff --git a/scripts/parse_syscalls.py b/scripts/parse_syscalls.py index 2d79b535b07..25c56223da8 100644 --- a/scripts/parse_syscalls.py +++ b/scripts/parse_syscalls.py @@ -5,7 +5,7 @@ # SPDX-License-Identifier: Apache-2.0 """ -Script to scan Zephyr include directories and emit system call metadata +Script to scan Zephyr include directories and emit system call and subsystem metadata System calls require a great deal of boilerplate code in order to implement completely. This script is the first step in the build system's process of @@ -26,7 +26,7 @@ import argparse import os import json -api_regex = re.compile(r''' +syscall_regex = re.compile(r''' __syscall\s+ # __syscall attribute, must be first ([^(]+) # type and name of system call (split later) [(] # Function opening parenthesis @@ -34,9 +34,16 @@ __syscall\s+ # __syscall attribute, must be first [)] # Closing parenthesis ''', re.MULTILINE | re.VERBOSE) +subsys_regex = re.compile(r''' +__subsystem\s+ # __subsystem attribute, must be first +struct\s+ # struct keyword is next +([^{]+) # name of subsystem +[{] # Open curly bracket +''', re.MULTILINE | re.VERBOSE) def analyze_headers(multiple_directories): - ret = [] + syscall_ret = [] + subsys_ret = [] for base_path in multiple_directories: for root, dirs, files in os.walk(base_path, topdown=True): @@ -44,23 +51,41 @@ def analyze_headers(multiple_directories): files.sort() for fn in files: - # toolchain/common.h has the definition of __syscall which we + # toolchain/common.h has the definitions of __syscall and __subsystem which we # don't want to trip over path = os.path.join(root, fn) if not fn.endswith(".h") or path.endswith(os.path.join(os.sep, 'toolchain', 'common.h')): continue with open(path, "r", encoding="utf-8") as fp: - try: - result = [(mo.groups(), fn) - for mo in api_regex.finditer(fp.read())] - except Exception: - sys.stderr.write("While parsing %s\n" % fn) - raise + contents = fp.read() - ret.extend(result) + try: + syscall_result = [(mo.groups(), fn) + for mo in syscall_regex.finditer(contents)] + subsys_result = [mo.groups()[0].strip() + for mo in subsys_regex.finditer(contents)] + except Exception: + sys.stderr.write("While parsing %s\n" % fn) + raise - return ret + syscall_ret.extend(syscall_result) + subsys_ret.extend(subsys_result) + + return syscall_ret, subsys_ret + + +def update_file_if_changed(path, new): + if os.path.exists(path): + with open(path, 'r') as fp: + old = fp.read() + + if new != old: + with open(path, 'w') as fp: + fp.write(new) + else: + with open(path, 'w') as fp: + fp.write(new) def parse_args(): @@ -76,34 +101,33 @@ def parse_args(): parser.add_argument( "-j", "--json-file", required=True, help="Write system call prototype information as json to file") + parser.add_argument( + "-s", "--subsystem-file", required=True, + help="Write subsystem name information as json to file") args = parser.parse_args() def main(): parse_args() - syscalls = analyze_headers(args.include) + syscalls, subsys = analyze_headers(args.include) + + # Only write json files if they don't exist or have changes since + # they will force and incremental rebuild. syscalls_in_json = json.dumps( syscalls, indent=4, sort_keys=True ) + update_file_if_changed(args.json_file, syscalls_in_json) - # Check if the file already exists, and if there are no changes, - # don't touch it since that will force an incremental rebuild - path = args.json_file - new = syscalls_in_json - if os.path.exists(path): - with open(path, 'r') as fp: - old = fp.read() - - if new != old: - with open(path, 'w') as fp: - fp.write(new) - else: - with open(path, 'w') as fp: - fp.write(new) + subsys_in_json = json.dumps( + subsys, + indent=4, + sort_keys=True + ) + update_file_if_changed(args.subsystem_file, subsys_in_json) if __name__ == "__main__":