michaelh/zephyr
1
0
Fork 0
Code Releases Activity

cmake: Re-organize syscall generation wrt. the build system

Browse source 
This commit fixes
https://github.com/zephyrproject-rtos/zephyr/issues/5008.

It does so by splitting up gen_syscalls.py into two scripts with a
json metadata file to communicate syscall metadata between them. The
parsing script parses header files from include/ and writes syscall
metadata to a file if the contents changed. The generation script
reads from the json file and generates syscall code.

The build system DAG now looks like this:

always_rebuild -> json -> syscalls -> offset.o

The script for generating json will do so only if the content changes,
this ensures that the entire DAG does not always do a full rebuild.

Signed-off-by: Sebastian Bøe <sebastian.boe@nordicsemi.no>
This commit is contained in:
Sebastian Bøe 2017-11-20 13:03:55 +01:00 • committed by Andrew Boie
commit 13a6840261
5 changed files with 203 additions and 136 deletions
Download patch file Download diff file Expand all files Collapse all files

39
CMakeLists.txt
View file

@ -302,6 +302,38 @@ add_custom_command( OUTPUT ${syscall_macros_h}
DEPENDS ${PROJECT_SOURCE_DIR}/scripts/gen_syscall_header.py
)
# This command is a hack to support commands that are always run. Any
# target that depends on always_rebuild will always be rebuilt.
add_custom_command(OUTPUT always_rebuild COMMAND cmake -E echo Building for board ${BOARD})
set(syscall_list_h ${CMAKE_CURRENT_BINARY_DIR}/include/generated/syscall_list.h)
set(syscalls_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls.json)
add_custom_command(
OUTPUT
${syscalls_json}
COMMAND
${PYTHON_EXECUTABLE}
${PROJECT_SOURCE_DIR}/scripts/parse_syscalls.py
--include ${PROJECT_SOURCE_DIR}/include # Read files from this dir
--json-file ${syscalls_json} # Write this file
DEPENDS always_rebuild
)
add_custom_target(syscall_list_h_target DEPENDS ${syscall_list_h})
add_custom_command(OUTPUT include/generated/syscall_dispatch.c ${syscall_list_h}
# Also, some files are written to include/generated/syscalls/
COMMAND
${PYTHON_EXECUTABLE}
${PROJECT_SOURCE_DIR}/scripts/gen_syscalls.py
--json-file ${syscalls_json} # Read this file
--base-output include/generated/syscalls # Write to this dir
--syscall-dispatch include/generated/syscall_dispatch.c # Write this file
> ${syscall_list_h} # Write stdout to this file
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
DEPENDS ${syscalls_json}
)
# Generate offsets.c.obj from offsets.c
# Generate offsets.h from offsets.c.obj
@ -309,9 +341,12 @@ set(OFFSETS_C_PATH $ENV{ZEPHYR_BASE}/arch/${ARCH}/core/offsets/offsets.c)
set(OFFSETS_O_PATH ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/offsets.dir/arch/${ARCH}/core/offsets/offsets.c.obj)
set(OFFSETS_H_PATH ${PROJECT_BINARY_DIR}/include/generated/offsets.h)
add_library(offsets STATIC ${OFFSETS_C_PATH})
add_library( offsets STATIC ${OFFSETS_C_PATH})
target_link_libraries(offsets zephyr_interface)
add_dependencies(offsets syscall_macros_h_target)
add_dependencies( offsets
syscall_list_h_target
syscall_macros_h_target
)
add_custom_command(
OUTPUT ${OFFSETS_H_PATH}

12
cmake/app/boilerplate.cmake
View file

@ -221,18 +221,6 @@ include(${BOARD_DIR}/board.cmake OPTIONAL)
zephyr_library_named(app)
execute_process(
COMMAND
${PYTHON_EXECUTABLE}
$ENV{ZEPHYR_BASE}/scripts/gen_syscalls.py
--include $ENV{ZEPHYR_BASE}/include # Read files from this dir
--base-output include/generated/syscalls # Write to this dir
--syscall-dispatch include/generated/syscall_dispatch.c # Write this file
INPUT_FILE kconfig/include/config/auto.conf # Read this file from stdin
OUTPUT_FILE include/generated/syscall_list.h # Write stdout to this file
WORKING_DIRECTORY ${ZEPHYR_BINARY_DIR}
)
add_subdirectory($ENV{ZEPHYR_BASE} ${__build_dir})
define_property(GLOBAL PROPERTY ZEPHYR_LIBS

8
doc/kernel/usermode/syscalls.rst
View file

@ -54,10 +54,10 @@ supervisor mode. For example, to initialize a semaphore:
The :c:macro:`__syscall` attribute is very special. To the C compiler, it
simply expands to 'static inline'. However to the post-build
``gen_syscalls.py`` script, it indicates that this API is a system call and
generates the body of the function. The ``gen_syscalls.py`` script does some
parsing of the function prototype, to determine the data types of its return
value and arguments, and has some limitations:
``parse_syscalls.py`` script, it indicates that this API is a system call.
The ``parse_syscalls.py`` script does some parsing of the function prototype,
to determine the data types of its return value and arguments, and has some
limitations:
* Array arguments must be passed in as pointers, not arrays. For example,
``int foo[]`` or ``int foo[12]`` is not allowed, but should instead be

124
scripts/gen_syscalls.py
View file

@ -8,112 +8,7 @@ import sys
import re
import argparse
import os
api_regex = re.compile(r'''
__syscall\s+ # __syscall attribute, must be first
([^(]+) # type and name of system call (split later)
[(] # Function opening parenthesis
([^)]*) # Arg list (split later)
[)] # Closing parenthesis
''', re.MULTILINE | re.VERBOSE)
typename_regex = re.compile(r'(.*?)([A-Za-z0-9_]+)$')
class SyscallParseException(Exception):
pass
def typename_split(item):
if "[" in item:
raise SyscallParseException("Please pass arrays to syscalls as pointers, unable to process '%s'"
% item)
if "(" in item:
raise SyscallParseException("Please use typedefs for function pointers")
mo = typename_regex.match(item)
if not mo:
raise SyscallParseException("Malformed system call invocation")
m = mo.groups()
return (m[0].strip(), m[1])
def analyze_fn(match_group, fn):
func, args = match_group
try:
if args == "void":
args = []
else:
args = [typename_split(a.strip()) for a in args.split(",")]
func_type, func_name = typename_split(func)
except SyscallParseException:
sys.stderr.write("In declaration of %s\n" % func)
raise
sys_id = "K_SYSCALL_" + func_name.upper()
if func_type == "void":
suffix = "_VOID"
is_void = True
else:
is_void = False
if func_type in ["s64_t", "u64_t"]:
suffix = "_RET64"
else:
suffix = ""
is_void = (func_type == "void")
# Get the proper system call macro invocation, which depends on the
# number of arguments, the return type, and whether the implementation
# is an inline function
macro = "K_SYSCALL_DECLARE%d%s" % (len(args), suffix)
# Flatten the argument lists and generate a comma separated list
# of t0, p0, t1, p1, ... tN, pN as expected by the macros
flat_args = [i for sublist in args for i in sublist]
if not is_void:
flat_args = [func_type] + flat_args
flat_args = [sys_id, func_name] + flat_args
argslist = ", ".join(flat_args)
invocation = "%s(%s);" % (macro, argslist)
handler = "_handler_" + func_name
# Entry in _k_syscall_table
table_entry = "[%s] = %s" % (sys_id, handler)
return (fn, handler, invocation, sys_id, table_entry)
def analyze_headers(base_path):
ret = []
for root, dirs, files in os.walk(base_path):
for fn in files:
# toolchain/common.h has the definition of __syscall which we
# don't want to trip over
path = os.path.join(root, fn)
if not fn.endswith(".h") or path.endswith("toolchain/common.h"):
continue
with open(path, "r", encoding="utf-8") as fp:
try:
result = [analyze_fn(mo.groups(), fn)
for mo in api_regex.finditer(fp.read())]
except Exception:
sys.stderr.write("While parsing %s\n" % fn)
raise
ret.extend(result)
return ret
import json
table_template = """/* auto-generated by gen_syscalls.py, don't edit */
@ -191,8 +86,8 @@ def parse_args():
parser = argparse.ArgumentParser(description = __doc__,
formatter_class = argparse.RawDescriptionHelpFormatter)
parser.add_argument("-i", "--include", required=True,
help="Base include directory")
parser.add_argument("-i", "--json-file", required=True,
help="Read syscall information from json file")
parser.add_argument("-d", "--syscall-dispatch", required=True,
help="output C system call dispatch table file")
parser.add_argument("-o", "--base-output", required=True,
@ -203,7 +98,9 @@ def parse_args():
def main():
parse_args()
syscalls = analyze_headers(args.include)
with open(args.json_file, 'r') as fd:
syscalls = json.load(fd)
invocations = {}
ids = []
table_entries = []
@ -237,15 +134,6 @@ def main():
header = syscall_template % "\n\n".join(invo_list)
# Check if the file already exists, and if there are no changes,
# don't touch it since that will force an incremental rebuild
if os.path.exists(out_fn):
with open(out_fn, "r") as fp:
old_data = fp.read()
if old_data == header:
continue
with open(out_fn, "w") as fp:
fp.write(header)

156
scripts/parse_syscalls.py Normal file
View file

@ -0,0 +1,156 @@
#!/usr/bin/env python3
#
# Copyright (c) 2017 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
import sys
import re
import argparse
import os
import json
api_regex = re.compile(r'''
__syscall\s+ # __syscall attribute, must be first
([^(]+) # type and name of system call (split later)
[(] # Function opening parenthesis
([^)]*) # Arg list (split later)
[)] # Closing parenthesis
''', re.MULTILINE | re.VERBOSE)
typename_regex = re.compile(r'(.*?)([A-Za-z0-9_]+)$')
class SyscallParseException(Exception):
pass
def typename_split(item):
if "[" in item:
raise SyscallParseException("Please pass arrays to syscalls as pointers, unable to process '%s'"
% item)
if "(" in item:
raise SyscallParseException("Please use typedefs for function pointers")
mo = typename_regex.match(item)
if not mo:
raise SyscallParseException("Malformed system call invocation")
m = mo.groups()
return (m[0].strip(), m[1])
def analyze_fn(match_group, fn):
func, args = match_group
try:
if args == "void":
args = []
else:
args = [typename_split(a.strip()) for a in args.split(",")]
func_type, func_name = typename_split(func)
except SyscallParseException:
sys.stderr.write("In declaration of %s\n" % func)
raise
sys_id = "K_SYSCALL_" + func_name.upper()
if func_type == "void":
suffix = "_VOID"
is_void = True
else:
is_void = False
if func_type in ["s64_t", "u64_t"]:
suffix = "_RET64"
else:
suffix = ""
is_void = (func_type == "void")
# Get the proper system call macro invocation, which depends on the
# number of arguments, the return type, and whether the implementation
# is an inline function
macro = "K_SYSCALL_DECLARE%d%s" % (len(args), suffix)
# Flatten the argument lists and generate a comma separated list
# of t0, p0, t1, p1, ... tN, pN as expected by the macros
flat_args = [i for sublist in args for i in sublist]
if not is_void:
flat_args = [func_type] + flat_args
flat_args = [sys_id, func_name] + flat_args
argslist = ", ".join(flat_args)
invocation = "%s(%s);" % (macro, argslist)
handler = "_handler_" + func_name
# Entry in _k_syscall_table
table_entry = "[%s] = %s" % (sys_id, handler)
return (fn, handler, invocation, sys_id, table_entry)
def analyze_headers(base_path):
ret = []
for root, dirs, files in os.walk(base_path):
for fn in files:
# toolchain/common.h has the definition of __syscall which we
# don't want to trip over
path = os.path.join(root, fn)
if not fn.endswith(".h") or path.endswith("toolchain/common.h"):
continue
with open(path, "r", encoding="utf-8") as fp:
try:
result = [analyze_fn(mo.groups(), fn)
for mo in api_regex.finditer(fp.read())]
except Exception:
sys.stderr.write("While parsing %s\n" % fn)
raise
ret.extend(result)
return ret
def parse_args():
global args
parser = argparse.ArgumentParser(description = __doc__,
formatter_class = argparse.RawDescriptionHelpFormatter)
parser.add_argument("-i", "--include", required=True,
help="Base include directory")
parser.add_argument("-j", "--json-file", required=True,
help="Write system call prototype information as json to file")
args = parser.parse_args()
def main():
parse_args()
syscalls = analyze_headers(args.include)
syscalls_in_json = json.dumps(
syscalls,
indent=4,
sort_keys=True
)
# Check if the file already exists, and if there are no changes,
# don't touch it since that will force an incremental rebuild
path = args.json_file
new = syscalls_in_json
if os.path.exists(path):
with open(path, 'r') as fp:
old = fp.read()
if new != old:
with open(path, 'w') as fp:
fp.write(new)
else:
with open(path, 'w') as fp:
fp.write(new)
if __name__ == "__main__":
main()