CMake/Kconfig: CMake and Kconfig build integration

This commit allows for Zephyr modules to be natively integrated into
the build system with CMakeLists.txt and Kconfig files.

The sourcing of module files are done in following order:
- If <module>/zephyr/module.yml exists, use cmake and kconfig settings
  for sourcing of additional file
- Else if <module>/zephyr/CMakeLists.txt exists, source this file into
  CMake build tree and add <module>/zephyr/Kconfig as osource

If none of the above files are present, the project is considered to
not be a Zephyr module

Signed-off-by: Torsten Rasmussen <torsten.rasmussen@nordicsemi.no>

CMakeLists.txt

@@ -552,6 +552,15 @@ add_subdirectory(subsys)
 add_subdirectory(drivers)
 add_subdirectory(tests)
 
+# Add all zephyr modules subdirectories.
+message("Including module(s): ${ZEPHYR_MODULES_NAME}")
+set(index 0)
+foreach(module_name ${ZEPHYR_MODULES_NAME})
+  list(GET ZEPHYR_MODULES_DIR ${index} module_dir)
+  math(EXPR index "${index} + 1")
+  add_subdirectory(${module_dir} ${CMAKE_BINARY_DIR}/${module_name})
+endforeach()
+
 set(syscall_macros_h ${ZEPHYR_BINARY_DIR}/include/generated/syscall_macros.h)
 
 add_custom_target(${SYSCALL_MACROS_H_TARGET} DEPENDS ${syscall_macros_h})

Kconfig.zephyr

@@ -40,6 +40,8 @@ source "ext/Kconfig"
 
 source "tests/Kconfig"
 
+source "$(PROJECT_BINARY_DIR)/Kconfig.modules"
+
 menu "Build and Link Features"
 
 menu "Linker Options"

cmake/app/boilerplate.cmake

@@ -396,6 +396,7 @@ include(${ZEPHYR_BASE}/cmake/host-tools.cmake)
 # preprocess DT sources, and then, after we have finished processing
 # both DT and Kconfig we complete the target-specific configuration,
 # and possibly change the toolchain.
+include(${ZEPHYR_BASE}/cmake/zephyr_module.cmake)
 include(${ZEPHYR_BASE}/cmake/generic_toolchain.cmake)
 include(${ZEPHYR_BASE}/cmake/dts.cmake)
 include(${ZEPHYR_BASE}/cmake/kconfig.cmake)

cmake/zephyr_module.cmake

@@ -0,0 +1,84 @@
+# This cmake file provides functionality to import additional out-of-tree, OoT
+# CMakeLists.txt and Kconfig files into Zephyr build system.
+# It uses -DZEPHYR_MODULES=<oot-path-to-module>[;<additional-oot-module(s)>]
+# given to CMake for a list of folders to search.
+# It looks for: <oot-module>/zephyr/module.yml or
+#               <oot-module>/zephyr/CMakeLists.txt
+# to load the oot-module into Zephyr build system.
+# If west is available, it uses `west list` to obtain a list of projects to
+# search for zephyr/module.yml
+
+if(ZEPHYR_MODULES)
+  set(west_project_list ${ZEPHYR_MODULES})
+elseif(WEST)
+  ## Use `west list` to fetch all west handled projects.
+  execute_process(
+    COMMAND
+    ${WEST} list --format={posixpath}
+    OUTPUT_VARIABLE
+    west_project_output
+  )
+  string(REGEX REPLACE "[\r\n]+" ";" west_project_list "${west_project_output}")
+endif()
+
+if(ZEPHYR_EXTRA_MODULES)
+  list(APPEND west_project_list ${ZEPHYR_EXTRA_MODULES})
+endif()
+
+# Clear the Kconfig.modules generated file in case modules has been removed.
+# The Kconfig.modules contains a list of additional Kconfig files to be sourced
+# based upon <module>/zephyr/module.yml files.
+file(WRITE ${PROJECT_BINARY_DIR}/Kconfig.modules
+  "# NOTE: THIS FILE IS AUTOGENERATED BY CMAKE\n"
+  )
+# For each west managed project, determine if the project is a zephyr module.
+foreach(module ${west_project_list})
+  set(cmake_subdir "zephyr")
+  if(${ZEPHYR_BASE} STREQUAL ${module})
+    # Ignore Zephyr project to avoid potential invalid looping
+  elseif(EXISTS "${module}/zephyr/module.yml")
+    set(kconfig_osource "osource \"${module}/zephyr/Kconfig\"\n")
+    get_filename_component(module_name ${module} NAME)
+    execute_process(
+      COMMAND
+      ${PYTHON_EXECUTABLE}
+      ${ZEPHYR_BASE}/scripts/yaml_to_cmake.py
+      -i "${module}/zephyr/module.yml"
+      -o "${CMAKE_CURRENT_BINARY_DIR}/zephyr_module_${module_name}.txt"
+      -s "build"
+      )
+    file(
+      STRINGS
+      "${CMAKE_CURRENT_BINARY_DIR}/zephyr_module_${module_name}.txt"
+      zephyr_module
+      )
+
+    foreach(key_value ${zephyr_module})
+      if(${key_value} MATCHES "^cmake=")
+        string(REGEX REPLACE "^cmake=" "" cmake_subdir ${key_value})
+      elseif(${key_value} MATCHES "^kconfig=")
+        string(
+          REGEX REPLACE
+          "^kconfig="
+          "osource \"${module}/"
+          kconfig_osource
+          ${key_value} "\"\n"
+          )
+      endif()
+    endforeach()
+
+    list(APPEND ZEPHYR_MODULES_NAME ${module_name})
+    list(APPEND ZEPHYR_MODULES_DIR ${module}/${cmake_subdir})
+    file(APPEND ${PROJECT_BINARY_DIR}/Kconfig.modules ${kconfig_osource})
+  elseif(EXISTS "${module}/${cmake_subdir}/CMakeLists.txt")
+    set(kconfig_osource "osource \"${module}/zephyr/Kconfig\"\n")
+    get_filename_component(module_name ${module} NAME)
+    list(APPEND ZEPHYR_MODULES_NAME ${module_name})
+    list(APPEND ZEPHYR_MODULES_DIR ${module}/${cmake_subdir})
+    file(APPEND ${PROJECT_BINARY_DIR}/Kconfig.modules ${kconfig_osource})
+  else()
+    # Not a Zephyr module, ignore.
+  endif()
+
+endforeach()
+

scripts/yaml_to_cmake.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2019, Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''Tool for a simple parsing of YAML files and return a ;-list of <key>=<value>
+pairs to use within a CMake build file.
+'''
+
+import argparse
+import sys
+import yaml
+import pykwalify.core
+
+
+METADATA_SCHEMA = '''\
+## A pykwalify schema for basic validation of the structure of a
+## metadata YAML file.
+##
+# The zephyr/module.yml file is a simple list of key value pairs to be used by
+# the build system.
+type: map
+mapping:
+  build:
+    required: true
+    type: map
+    mapping:
+      cmake:
+        required: false
+        type: str
+      kconfig:
+        required: false
+        type: str
+'''
+
+
+def main():
+    parser = argparse.ArgumentParser(description='''
+    Converts YAML to a CMake list''')
+
+    parser.add_argument('-i', '--input', required=True,
+                        help='YAML file with data')
+    parser.add_argument('-o', '--output', required=True,
+                        help='File to write with CMake data')
+    parser.add_argument('-s', '--section', required=True,
+                        help='Section in YAML file to parse')
+    args = parser.parse_args()
+
+    with open(args.input, 'r') as f:
+        meta = yaml.safe_load(f.read())
+
+    pykwalify.core.Core(source_data=meta,
+                        schema_data=yaml.safe_load(METADATA_SCHEMA)).validate()
+
+    val_str = ''
+
+    section = meta.get(args.section)
+    if section is not None:
+        for key in section:
+            val_str += '{}={}\n'.format(key, section[key])
+
+    with open(args.output, 'w') as f:
+        f.write(val_str)
+
+
+if __name__ == "__main__":
+    main()