scripts: Dynamically add driver subsystems to subsystems list

This change extends the parse_syscalls.py script to scan for a
__subsystem sentinal added to driver api declarations. It thens
generates a list that is passed into gen_kobject_list.py to extend
the subsystems list. This allows subsystems to be declared in the
code instead of a separate python list and provides a mechanism for
defining out-of-tree subsystems.

Signed-off-by: Corey Wharton <coreyw7@fb.com>

CMakeLists.txt

@@ -494,6 +494,7 @@ endif()
 
 set(syscall_list_h ${CMAKE_CURRENT_BINARY_DIR}/include/generated/syscall_list.h)
 set(syscalls_json  ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls.json)
+set(subsys_json    ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/subsystems.json)
 
 # The syscalls subdirs txt file is constructed by python containing a list of folders to use for
 # dependency handling, including empty folders.
@@ -585,12 +586,14 @@ endforeach()
 add_custom_command(
   OUTPUT
   ${syscalls_json}
+  ${subsys_json}
   COMMAND
   ${PYTHON_EXECUTABLE}
   ${ZEPHYR_BASE}/scripts/parse_syscalls.py
    --include          ${ZEPHYR_BASE}/include        # Read files from this dir
   ${parse_syscalls_include_args}                    # Read files from these dirs also
-  --json-file        ${syscalls_json}              # Write this file
+  --json-file        ${syscalls_json}               # Write this file
+  --subsystem-file   ${subsys_json}                 # Write subsystem list to this file
   DEPENDS ${syscalls_subdirs_trigger} ${PARSE_SYSCALLS_HEADER_DEPENDS}
   )
 
@@ -617,6 +620,9 @@ add_custom_command(OUTPUT include/generated/syscall_dispatch.c ${syscall_list_h}
   DEPENDS ${syscalls_json}
   )
 
+# This is passed into all calls to the gen_kobject_list.py script.
+set(gen_kobject_list_include_args --include ${subsys_json})
+
 set(DRV_VALIDATION ${PROJECT_BINARY_DIR}/include/generated/driver-validation.h)
 add_custom_command(
   OUTPUT ${DRV_VALIDATION}
@@ -624,8 +630,11 @@ add_custom_command(
   ${PYTHON_EXECUTABLE}
   ${ZEPHYR_BASE}/scripts/gen_kobject_list.py
   --validation-output ${DRV_VALIDATION}
+  ${gen_kobject_list_include_args}
   $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:--verbose>
-  DEPENDS ${ZEPHYR_BASE}/scripts/gen_kobject_list.py
+  DEPENDS
+  ${ZEPHYR_BASE}/scripts/gen_kobject_list.py
+  ${subsys_json}
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   )
 add_custom_target(${DRIVER_VALIDATION_H_TARGET} DEPENDS ${DRV_VALIDATION})
@@ -941,8 +950,11 @@ if(CONFIG_USERSPACE)
     ${GEN_KOBJ_LIST}
     --kernel $<TARGET_FILE:${ZEPHYR_PREBUILT_EXECUTABLE}>
     --gperf-output ${OBJ_LIST}
+    ${gen_kobject_list_include_args}
     $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:--verbose>
-    DEPENDS ${ZEPHYR_PREBUILT_EXECUTABLE}
+    DEPENDS
+    ${ZEPHYR_PREBUILT_EXECUTABLE}
+    ${subsys_json}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     )
   add_custom_target(obj_list DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${OBJ_LIST})

cmake/kobj.cmake

@@ -21,8 +21,11 @@ function(gen_kobj gen_dir_out)
     --kobj-types-output ${KOBJ_TYPES}
     --kobj-otype-output ${KOBJ_OTYPE}
     --kobj-size-output ${KOBJ_SIZE}
+    ${gen_kobject_list_include_args}
     $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:--verbose>
-    DEPENDS $ENV{ZEPHYR_BASE}/scripts/gen_kobject_list.py
+    DEPENDS
+    $ENV{ZEPHYR_BASE}/scripts/gen_kobject_list.py
+    ${subsys_json}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     )
   add_custom_target(${KOBJ_TYPES_H_TARGET} DEPENDS ${KOBJ_TYPES} ${KOBJ_OTYPE})

include/toolchain/common.h

@@ -132,6 +132,11 @@
 #define __syscall
 #endif /* #ifndef ZTEST_UNITTEST */
 
+/* Used as a sentinel by parse_syscalls.py to identify what API structs
+ * define driver subsystems.
+ */
+#define __subsystem
+
 #ifndef BUILD_ASSERT
 /* compile-time assertion that makes the build fail */
 #define BUILD_ASSERT(EXPR) \

scripts/gen_kobject_list.py

@@ -56,6 +56,7 @@ import argparse
 import math
 import os
 import struct
+import json
 from elf_helper import ElfHelper, kobject_to_enum
 
 from collections import OrderedDict
@@ -90,8 +91,6 @@ kobjects = OrderedDict([
     ("k_futex", (None, True))
 ])
 
-
-
 subsystems = [
     "adc_driver_api",
     "aio_cmp_driver_api",
@@ -331,6 +330,11 @@ def write_kobj_size_output(fp):
             fp.write("#endif\n")
 
 
+def parse_subsystems_list_file(path):
+    with open(path, "r") as fp:
+        subsys_list = json.load(fp)
+    subsystems.extend(subsys_list)
+
 def parse_args():
     global args
 
@@ -355,6 +359,11 @@ def parse_args():
     parser.add_argument(
         "-Z", "--kobj-size-output", required=False,
         help="Output case statements for obj_size_get()")
+    parser.add_argument("-i", "--include-subsystem-list", required=False, action='append',
+        help='''Specifies a file with a JSON encoded list of subsystem names to append to
+        the driver subsystems list. Can be specified multiple times:
+        -i file1 -i file2 ...''')
+
     parser.add_argument("-v", "--verbose", action="store_true",
                         help="Print extra debugging information")
     args = parser.parse_args()
@@ -365,6 +374,10 @@ def parse_args():
 def main():
     parse_args()
 
+    if args.include_subsystem_list is not None:
+        for list_file in args.include_subsystem_list:
+            parse_subsystems_list_file(list_file)
+
     if args.gperf_output:
         assert args.kernel, "--kernel ELF required for --gperf-output"
         eh = ElfHelper(args.kernel, args.verbose, kobjects, subsystems)

scripts/parse_syscalls.py

@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 """
-Script to scan Zephyr include directories and emit system call metadata
+Script to scan Zephyr include directories and emit system call and subsystem metadata
 
 System calls require a great deal of boilerplate code in order to implement
 completely. This script is the first step in the build system's process of
@@ -26,7 +26,7 @@ import argparse
 import os
 import json
 
-api_regex = re.compile(r'''
+syscall_regex = re.compile(r'''
 __syscall\s+                    # __syscall attribute, must be first
 ([^(]+)                         # type and name of system call (split later)
 [(]                             # Function opening parenthesis
@@ -34,9 +34,16 @@ __syscall\s+                    # __syscall attribute, must be first
 [)]                             # Closing parenthesis
 ''', re.MULTILINE | re.VERBOSE)
 
+subsys_regex = re.compile(r'''
+__subsystem\s+                  # __subsystem attribute, must be first
+struct\s+                       # struct keyword is next
+([^{]+)                         # name of subsystem
+[{]                             # Open curly bracket
+''', re.MULTILINE | re.VERBOSE)
 
 def analyze_headers(multiple_directories):
-    ret = []
+    syscall_ret = []
+    subsys_ret = []
 
     for base_path in multiple_directories:
         for root, dirs, files in os.walk(base_path, topdown=True):
@@ -44,23 +51,41 @@ def analyze_headers(multiple_directories):
             files.sort()
             for fn in files:
 
-                # toolchain/common.h has the definition of __syscall which we
+                # toolchain/common.h has the definitions of __syscall and __subsystem which we
                 # don't want to trip over
                 path = os.path.join(root, fn)
                 if not fn.endswith(".h") or path.endswith(os.path.join(os.sep, 'toolchain', 'common.h')):
                     continue
 
                 with open(path, "r", encoding="utf-8") as fp:
-                    try:
+                    contents = fp.read()
-                        result = [(mo.groups(), fn)
-                                  for mo in api_regex.finditer(fp.read())]
-                    except Exception:
-                        sys.stderr.write("While parsing %s\n" % fn)
-                        raise
 
-                    ret.extend(result)
+                try:
+                    syscall_result = [(mo.groups(), fn)
+                                      for mo in syscall_regex.finditer(contents)]
+                    subsys_result = [mo.groups()[0].strip()
+                                     for mo in subsys_regex.finditer(contents)]
+                except Exception:
+                    sys.stderr.write("While parsing %s\n" % fn)
+                    raise
 
-    return ret
+                syscall_ret.extend(syscall_result)
+                subsys_ret.extend(subsys_result)
+
+    return syscall_ret, subsys_ret
+
+
+def update_file_if_changed(path, new):
+    if os.path.exists(path):
+        with open(path, 'r') as fp:
+            old = fp.read()
+
+        if new != old:
+            with open(path, 'w') as fp:
+                fp.write(new)
+    else:
+        with open(path, 'w') as fp:
+            fp.write(new)
 
 
 def parse_args():
@@ -76,34 +101,33 @@ def parse_args():
     parser.add_argument(
         "-j", "--json-file", required=True,
         help="Write system call prototype information as json to file")
+    parser.add_argument(
+        "-s", "--subsystem-file", required=True,
+        help="Write subsystem name information as json to file")
     args = parser.parse_args()
 
 
 def main():
     parse_args()
 
-    syscalls = analyze_headers(args.include)
+    syscalls, subsys = analyze_headers(args.include)
+
+    # Only write json files if they don't exist or have changes since
+    # they will force and incremental rebuild.
 
     syscalls_in_json = json.dumps(
         syscalls,
         indent=4,
         sort_keys=True
     )
+    update_file_if_changed(args.json_file, syscalls_in_json)
 
-    # Check if the file already exists, and if there are no changes,
+    subsys_in_json = json.dumps(
-    # don't touch it since that will force an incremental rebuild
+        subsys,
-    path = args.json_file
+        indent=4,
-    new = syscalls_in_json
+        sort_keys=True
-    if os.path.exists(path):
+    )
-        with open(path, 'r') as fp:
+    update_file_if_changed(args.subsystem_file, subsys_in_json)
-            old = fp.read()
-
-        if new != old:
-            with open(path, 'w') as fp:
-                fp.write(new)
-    else:
-        with open(path, 'w') as fp:
-            fp.write(new)
 
 
 if __name__ == "__main__":