cmake: Re-organize syscall generation wrt. the build system

This commit fixes
https://github.com/zephyrproject-rtos/zephyr/issues/5008.

It does so by splitting up gen_syscalls.py into two scripts with a
json metadata file to communicate syscall metadata between them. The
parsing script parses header files from include/ and writes syscall
metadata to a file if the contents changed. The generation script
reads from the json file and generates syscall code.

The build system DAG now looks like this:

always_rebuild -> json -> syscalls -> offset.o

The script for generating json will do so only if the content changes,
this ensures that the entire DAG does not always do a full rebuild.

Signed-off-by: Sebastian Bøe <sebastian.boe@nordicsemi.no>

CMakeLists.txt

@@ -302,6 +302,38 @@ add_custom_command(                       OUTPUT  ${syscall_macros_h}
   DEPENDS ${PROJECT_SOURCE_DIR}/scripts/gen_syscall_header.py
   )
 
+# This command is a hack to support commands that are always run. Any
+# target that depends on always_rebuild will always be rebuilt.
+add_custom_command(OUTPUT always_rebuild COMMAND cmake -E echo Building for board ${BOARD})
+
+set(syscall_list_h ${CMAKE_CURRENT_BINARY_DIR}/include/generated/syscall_list.h)
+set(syscalls_json  ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls.json)
+
+add_custom_command(
+  OUTPUT
+  ${syscalls_json}
+  COMMAND
+  ${PYTHON_EXECUTABLE}
+  ${PROJECT_SOURCE_DIR}/scripts/parse_syscalls.py
+  --include          ${PROJECT_SOURCE_DIR}/include        # Read files from this dir
+  --json-file        ${syscalls_json}                     # Write this file
+  DEPENDS always_rebuild
+  )
+
+add_custom_target(syscall_list_h_target DEPENDS                ${syscall_list_h})
+add_custom_command(OUTPUT include/generated/syscall_dispatch.c ${syscall_list_h}
+  # Also, some files are written to include/generated/syscalls/
+  COMMAND
+  ${PYTHON_EXECUTABLE}
+  ${PROJECT_SOURCE_DIR}/scripts/gen_syscalls.py
+  --json-file        ${syscalls_json}                     # Read this file
+  --base-output      include/generated/syscalls           # Write to this dir
+  --syscall-dispatch include/generated/syscall_dispatch.c # Write this file
+  > ${syscall_list_h}                                     # Write stdout to this file
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS ${syscalls_json}
+  )
+
 # Generate offsets.c.obj from offsets.c
 # Generate offsets.h     from offsets.c.obj
 
@@ -311,7 +343,10 @@ set(OFFSETS_H_PATH ${PROJECT_BINARY_DIR}/include/generated/offsets.h)
 
 add_library(          offsets STATIC ${OFFSETS_C_PATH})
 target_link_libraries(offsets zephyr_interface)
-add_dependencies(offsets syscall_macros_h_target)
+add_dependencies(     offsets
+  syscall_list_h_target
+  syscall_macros_h_target
+  )
 
 add_custom_command(
   OUTPUT ${OFFSETS_H_PATH}

cmake/app/boilerplate.cmake

@@ -221,18 +221,6 @@ include(${BOARD_DIR}/board.cmake OPTIONAL)
 
 zephyr_library_named(app)
 
-execute_process(
-  COMMAND
-  ${PYTHON_EXECUTABLE}
-  $ENV{ZEPHYR_BASE}/scripts/gen_syscalls.py
-  --include          $ENV{ZEPHYR_BASE}/include            # Read files from this dir
-  --base-output      include/generated/syscalls           # Write to this dir
-  --syscall-dispatch include/generated/syscall_dispatch.c # Write this file
-  INPUT_FILE         kconfig/include/config/auto.conf     # Read this file from stdin
-  OUTPUT_FILE        include/generated/syscall_list.h     # Write stdout to this file
-  WORKING_DIRECTORY ${ZEPHYR_BINARY_DIR}
-  )
-
 add_subdirectory($ENV{ZEPHYR_BASE} ${__build_dir})
 
 define_property(GLOBAL PROPERTY ZEPHYR_LIBS

doc/kernel/usermode/syscalls.rst

@@ -54,10 +54,10 @@ supervisor mode. For example, to initialize a semaphore:
 
 The :c:macro:`__syscall` attribute is very special. To the C compiler, it
 simply expands to 'static inline'. However to the post-build
-``gen_syscalls.py`` script, it indicates that this API is a system call and
+``parse_syscalls.py`` script, it indicates that this API is a system call.
-generates the body of the function.  The ``gen_syscalls.py`` script does some
+The ``parse_syscalls.py`` script does some parsing of the function prototype,
-parsing of the function prototype, to determine the data types of its return
+to determine the data types of its return value and arguments, and has some
-value and arguments, and has some limitations:
+limitations:
 
 * Array arguments must be passed in as pointers, not arrays. For example,
   ``int foo[]`` or ``int foo[12]`` is not allowed, but should instead be

scripts/gen_syscalls.py

@@ -8,112 +8,7 @@ import sys
 import re
 import argparse
 import os
-
+import json
-
-api_regex = re.compile(r'''
-__syscall\s+                    # __syscall attribute, must be first
-([^(]+)                         # type and name of system call (split later)
-[(]                             # Function opening parenthesis
-([^)]*)                         # Arg list (split later)
-[)]                             # Closing parenthesis
-''', re.MULTILINE | re.VERBOSE)
-
-typename_regex = re.compile(r'(.*?)([A-Za-z0-9_]+)$')
-
-class SyscallParseException(Exception):
-    pass
-
-
-def typename_split(item):
-    if "[" in item:
-        raise SyscallParseException("Please pass arrays to syscalls as pointers, unable to process '%s'"
-                % item)
-
-    if "(" in item:
-        raise SyscallParseException("Please use typedefs for function pointers")
-
-    mo = typename_regex.match(item)
-    if not mo:
-        raise SyscallParseException("Malformed system call invocation")
-
-    m = mo.groups()
-    return (m[0].strip(), m[1])
-
-
-def analyze_fn(match_group, fn):
-    func, args = match_group
-
-    try:
-        if args == "void":
-            args = []
-        else:
-            args = [typename_split(a.strip()) for a in args.split(",")]
-
-        func_type, func_name = typename_split(func)
-    except SyscallParseException:
-        sys.stderr.write("In declaration of %s\n" % func)
-        raise
-
-    sys_id = "K_SYSCALL_" + func_name.upper()
-
-    if func_type == "void":
-        suffix = "_VOID"
-        is_void = True
-    else:
-        is_void = False
-        if func_type in ["s64_t", "u64_t"]:
-            suffix = "_RET64"
-        else:
-            suffix = ""
-
-    is_void = (func_type == "void")
-
-    # Get the proper system call macro invocation, which depends on the
-    # number of arguments, the return type, and whether the implementation
-    # is an inline function
-    macro = "K_SYSCALL_DECLARE%d%s" % (len(args), suffix)
-
-    # Flatten the argument lists and generate a comma separated list
-    # of t0, p0, t1, p1, ... tN, pN as expected by the macros
-    flat_args = [i for sublist in args for i in sublist]
-    if not is_void:
-        flat_args = [func_type] + flat_args
-    flat_args = [sys_id, func_name] + flat_args
-    argslist = ", ".join(flat_args)
-
-    invocation = "%s(%s);" % (macro, argslist)
-
-    handler = "_handler_" + func_name
-
-    # Entry in _k_syscall_table
-    table_entry = "[%s] = %s" % (sys_id, handler)
-
-    return (fn, handler, invocation, sys_id, table_entry)
-
-
-def analyze_headers(base_path):
-    ret = []
-
-    for root, dirs, files in os.walk(base_path):
-        for fn in files:
-
-            # toolchain/common.h has the definition of __syscall which we
-            # don't want to trip over
-            path = os.path.join(root, fn)
-            if not fn.endswith(".h") or path.endswith("toolchain/common.h"):
-                continue
-
-            with open(path, "r", encoding="utf-8") as fp:
-                try:
-                    result = [analyze_fn(mo.groups(), fn)
-                              for mo in api_regex.finditer(fp.read())]
-                except Exception:
-                    sys.stderr.write("While parsing %s\n" % fn)
-                    raise
-
-            ret.extend(result)
-
-    return ret
 
 table_template = """/* auto-generated by gen_syscalls.py, don't edit */
 
@@ -191,8 +86,8 @@ def parse_args():
     parser = argparse.ArgumentParser(description = __doc__,
             formatter_class = argparse.RawDescriptionHelpFormatter)
 
-    parser.add_argument("-i", "--include", required=True,
+    parser.add_argument("-i", "--json-file", required=True,
-            help="Base include directory")
+            help="Read syscall information from json file")
     parser.add_argument("-d", "--syscall-dispatch", required=True,
             help="output C system call dispatch table file")
     parser.add_argument("-o", "--base-output", required=True,
@@ -203,7 +98,9 @@ def parse_args():
 def main():
     parse_args()
 
-    syscalls = analyze_headers(args.include)
+    with open(args.json_file, 'r') as fd:
+        syscalls = json.load(fd)
+
     invocations = {}
     ids = []
     table_entries = []
@@ -237,15 +134,6 @@ def main():
 
         header = syscall_template % "\n\n".join(invo_list)
 
-        # Check if the file already exists, and if there are no changes,
-        # don't touch it since that will force an incremental rebuild
-        if os.path.exists(out_fn):
-            with open(out_fn, "r") as fp:
-                old_data = fp.read()
-
-            if old_data == header:
-                continue
-
         with open(out_fn, "w") as fp:
             fp.write(header)
 

scripts/parse_syscalls.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2017 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+import re
+import argparse
+import os
+import json
+
+api_regex = re.compile(r'''
+__syscall\s+                    # __syscall attribute, must be first
+([^(]+)                         # type and name of system call (split later)
+[(]                             # Function opening parenthesis
+([^)]*)                         # Arg list (split later)
+[)]                             # Closing parenthesis
+''', re.MULTILINE | re.VERBOSE)
+
+typename_regex = re.compile(r'(.*?)([A-Za-z0-9_]+)$')
+
+class SyscallParseException(Exception):
+    pass
+
+
+def typename_split(item):
+    if "[" in item:
+        raise SyscallParseException("Please pass arrays to syscalls as pointers, unable to process '%s'"
+                % item)
+
+    if "(" in item:
+        raise SyscallParseException("Please use typedefs for function pointers")
+
+    mo = typename_regex.match(item)
+    if not mo:
+        raise SyscallParseException("Malformed system call invocation")
+
+    m = mo.groups()
+    return (m[0].strip(), m[1])
+
+
+def analyze_fn(match_group, fn):
+    func, args = match_group
+
+    try:
+        if args == "void":
+            args = []
+        else:
+            args = [typename_split(a.strip()) for a in args.split(",")]
+
+        func_type, func_name = typename_split(func)
+    except SyscallParseException:
+        sys.stderr.write("In declaration of %s\n" % func)
+        raise
+
+    sys_id = "K_SYSCALL_" + func_name.upper()
+
+    if func_type == "void":
+        suffix = "_VOID"
+        is_void = True
+    else:
+        is_void = False
+        if func_type in ["s64_t", "u64_t"]:
+            suffix = "_RET64"
+        else:
+            suffix = ""
+
+    is_void = (func_type == "void")
+
+    # Get the proper system call macro invocation, which depends on the
+    # number of arguments, the return type, and whether the implementation
+    # is an inline function
+    macro = "K_SYSCALL_DECLARE%d%s" % (len(args), suffix)
+
+    # Flatten the argument lists and generate a comma separated list
+    # of t0, p0, t1, p1, ... tN, pN as expected by the macros
+    flat_args = [i for sublist in args for i in sublist]
+    if not is_void:
+        flat_args = [func_type] + flat_args
+    flat_args = [sys_id, func_name] + flat_args
+    argslist = ", ".join(flat_args)
+
+    invocation = "%s(%s);" % (macro, argslist)
+
+    handler = "_handler_" + func_name
+
+    # Entry in _k_syscall_table
+    table_entry = "[%s] = %s" % (sys_id, handler)
+
+    return (fn, handler, invocation, sys_id, table_entry)
+
+
+def analyze_headers(base_path):
+    ret = []
+
+    for root, dirs, files in os.walk(base_path):
+        for fn in files:
+
+            # toolchain/common.h has the definition of __syscall which we
+            # don't want to trip over
+            path = os.path.join(root, fn)
+            if not fn.endswith(".h") or path.endswith("toolchain/common.h"):
+                continue
+
+            with open(path, "r", encoding="utf-8") as fp:
+                try:
+                    result = [analyze_fn(mo.groups(), fn)
+                              for mo in api_regex.finditer(fp.read())]
+                except Exception:
+                    sys.stderr.write("While parsing %s\n" % fn)
+                    raise
+
+            ret.extend(result)
+
+    return ret
+
+def parse_args():
+    global args
+    parser = argparse.ArgumentParser(description = __doc__,
+            formatter_class = argparse.RawDescriptionHelpFormatter)
+
+    parser.add_argument("-i", "--include", required=True,
+            help="Base include directory")
+    parser.add_argument("-j", "--json-file", required=True,
+            help="Write system call prototype information as json to file")
+    args = parser.parse_args()
+
+def main():
+    parse_args()
+
+    syscalls = analyze_headers(args.include)
+
+    syscalls_in_json = json.dumps(
+        syscalls,
+        indent=4,
+        sort_keys=True
+    )
+
+    # Check if the file already exists, and if there are no changes,
+    # don't touch it since that will force an incremental rebuild
+    path = args.json_file
+    new = syscalls_in_json
+    if os.path.exists(path):
+        with open(path, 'r') as fp:
+            old = fp.read()
+
+        if new != old:
+            with open(path, 'w') as fp:
+                fp.write(new)
+    else:
+        with open(path, 'w') as fp:
+            fp.write(new)
+
+if __name__ == "__main__":
+    main()