From 5960119f16da5cc1da3505cd9235dcb1162b3844 Mon Sep 17 00:00:00 2001 From: Andrew Boie Date: Fri, 29 May 2020 13:24:51 -0700 Subject: [PATCH] scripts: parse_syscalls: generalize struct tags Now we can build up lists of data structures matching a list of particular tags, with __subsystem being just one case. Relax searches to also look inside C files, since struct prototypes may be declared there as well. Signed-off-by: Andrew Boie --- CMakeLists.txt | 16 +++++------ scripts/gen_kobject_list.py | 2 +- scripts/parse_syscalls.py | 57 ++++++++++++++++++++++++------------- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ee341157ce..02e8ba23d69 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -495,9 +495,9 @@ if(EXISTS ${CMAKE_BINARY_DIR}/zephyr_modules.txt) set(ZEPHYR_CURRENT_MODULE_DIR) endif() -set(syscall_list_h ${CMAKE_CURRENT_BINARY_DIR}/include/generated/syscall_list.h) -set(syscalls_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls.json) -set(subsys_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/subsystems.json) +set(syscall_list_h ${CMAKE_CURRENT_BINARY_DIR}/include/generated/syscall_list.h) +set(syscalls_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls.json) +set(struct_tags_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/struct_tags.json) # The syscalls subdirs txt file is constructed by python containing a list of folders to use for # dependency handling, including empty folders. @@ -589,20 +589,20 @@ endforeach() add_custom_command( OUTPUT ${syscalls_json} - ${subsys_json} + ${struct_tags_json} COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/parse_syscalls.py --include ${ZEPHYR_BASE}/include # Read files from this dir ${parse_syscalls_include_args} # Read files from these dirs also --json-file ${syscalls_json} # Write this file - --subsystem-file ${subsys_json} # Write subsystem list to this file + --tag-struct-file ${struct_tags_json} # Write subsystem list to this file DEPENDS ${syscalls_subdirs_trigger} ${PARSE_SYSCALLS_HEADER_DEPENDS} ) add_custom_target(${SYSCALL_LIST_H_TARGET} DEPENDS ${syscall_list_h}) add_custom_target(${PARSE_SYSCALLS_TARGET} - DEPENDS ${syscalls_json} ${subsys_json}) + DEPENDS ${syscalls_json} ${struct_tags_json}) # 64-bit systems do not require special handling of 64-bit system call # parameters or return values, indicate this to the system call boilerplate @@ -632,7 +632,7 @@ add_custom_command(OUTPUT include/generated/syscall_dispatch.c ${syscall_list_h} ) # This is passed into all calls to the gen_kobject_list.py script. -set(gen_kobject_list_include_args --include ${subsys_json}) +set(gen_kobject_list_include_args --include ${struct_tags_json}) set(DRV_VALIDATION ${PROJECT_BINARY_DIR}/include/generated/driver-validation.h) add_custom_command( @@ -646,7 +646,7 @@ add_custom_command( DEPENDS ${ZEPHYR_BASE}/scripts/gen_kobject_list.py ${PARSE_SYSCALLS_TARGET} - ${subsys_json} + ${struct_tags_json} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) add_custom_target(${DRIVER_VALIDATION_H_TARGET} DEPENDS ${DRV_VALIDATION}) diff --git a/scripts/gen_kobject_list.py b/scripts/gen_kobject_list.py index bf16135203f..54a4f0da098 100755 --- a/scripts/gen_kobject_list.py +++ b/scripts/gen_kobject_list.py @@ -918,7 +918,7 @@ def write_kobj_size_output(fp): def parse_subsystems_list_file(path): with open(path, "r") as fp: subsys_list = json.load(fp) - subsystems.extend(subsys_list) + subsystems.extend(subsys_list["__subsystem"]) def parse_args(): global args diff --git a/scripts/parse_syscalls.py b/scripts/parse_syscalls.py index 25c56223da8..f3bddefc349 100644 --- a/scripts/parse_syscalls.py +++ b/scripts/parse_syscalls.py @@ -10,10 +10,14 @@ Script to scan Zephyr include directories and emit system call and subsystem met System calls require a great deal of boilerplate code in order to implement completely. This script is the first step in the build system's process of auto-generating this code by doing a text scan of directories containing -header files, and building up a database of system calls and their +C or header files, and building up a database of system calls and their function call prototypes. This information is emitted to a generated JSON file for further processing. +This script also scans for struct definitions such as __subsystem and +__net_socket, emitting a JSON dictionary mapping tags to all the struct +declarations found that were tagged with them. + If the output JSON file already exists, its contents are checked against what information this script would have outputted; if the result is that the file would be unchanged, it is not modified to prevent unnecessary @@ -26,24 +30,37 @@ import argparse import os import json +regex_flags = re.MULTILINE | re.VERBOSE + syscall_regex = re.compile(r''' __syscall\s+ # __syscall attribute, must be first ([^(]+) # type and name of system call (split later) [(] # Function opening parenthesis ([^)]*) # Arg list (split later) [)] # Closing parenthesis -''', re.MULTILINE | re.VERBOSE) +''', regex_flags) -subsys_regex = re.compile(r''' -__subsystem\s+ # __subsystem attribute, must be first +struct_tags = ["__subsystem"] + +tagged_struct_decl_template = r''' +%s\s+ # tag, must be first struct\s+ # struct keyword is next ([^{]+) # name of subsystem [{] # Open curly bracket -''', re.MULTILINE | re.VERBOSE) +''' + +def tagged_struct_update(target_list, tag, contents): + regex = re.compile(tagged_struct_decl_template % tag, regex_flags) + items = [mo.groups()[0].strip() for mo in regex.finditer(contents)] + target_list.extend(items) + def analyze_headers(multiple_directories): syscall_ret = [] - subsys_ret = [] + tagged_ret = {} + + for tag in struct_tags: + tagged_ret[tag] = [] for base_path in multiple_directories: for root, dirs, files in os.walk(base_path, topdown=True): @@ -51,10 +68,12 @@ def analyze_headers(multiple_directories): files.sort() for fn in files: - # toolchain/common.h has the definitions of __syscall and __subsystem which we + # toolchain/common.h has the definitions of these tagswhich we # don't want to trip over path = os.path.join(root, fn) - if not fn.endswith(".h") or path.endswith(os.path.join(os.sep, 'toolchain', 'common.h')): + if (not (path.endswith(".h") or path.endswith(".c")) or + path.endswith(os.path.join(os.sep, 'toolchain', + 'common.h'))): continue with open(path, "r", encoding="utf-8") as fp: @@ -63,16 +82,15 @@ def analyze_headers(multiple_directories): try: syscall_result = [(mo.groups(), fn) for mo in syscall_regex.finditer(contents)] - subsys_result = [mo.groups()[0].strip() - for mo in subsys_regex.finditer(contents)] + for tag in struct_tags: + tagged_struct_update(tagged_ret[tag], tag, contents) except Exception: sys.stderr.write("While parsing %s\n" % fn) raise syscall_ret.extend(syscall_result) - subsys_ret.extend(subsys_result) - return syscall_ret, subsys_ret + return syscall_ret, tagged_ret def update_file_if_changed(path, new): @@ -102,18 +120,19 @@ def parse_args(): "-j", "--json-file", required=True, help="Write system call prototype information as json to file") parser.add_argument( - "-s", "--subsystem-file", required=True, - help="Write subsystem name information as json to file") + "-t", "--tag-struct-file", required=True, + help="Write tagged struct name information as json to file") + args = parser.parse_args() def main(): parse_args() - syscalls, subsys = analyze_headers(args.include) + syscalls, tagged = analyze_headers(args.include) # Only write json files if they don't exist or have changes since - # they will force and incremental rebuild. + # they will force an incremental rebuild. syscalls_in_json = json.dumps( syscalls, @@ -122,12 +141,12 @@ def main(): ) update_file_if_changed(args.json_file, syscalls_in_json) - subsys_in_json = json.dumps( - subsys, + tagged_struct_in_json = json.dumps( + tagged, indent=4, sort_keys=True ) - update_file_if_changed(args.subsystem_file, subsys_in_json) + update_file_if_changed(args.tag_struct_file, tagged_struct_in_json) if __name__ == "__main__":