scripts: support compile_commands.json in gen_app_partitions.py

Fixes: #40590

This commit updates gen_app_partitions.py to include only files present
in the current build by extracting the information from the CMake
generated `compile_commands.json` file.

This ensures that object files in sub-projects, such as `empty_cpu0`,
will not be considered by the script.

Using the compile_commands.json instead of walking the whole build tree
for finding object files also improves performance:

Time of executing `gen_app_partitions.py` (Old):
__________________________
Executed in  480.06 millis
   usr time  425.83 millis
   sys time   49.55 millis

Time of executing `gen_app_partitions.py` (New):
________________________________________________________
Executed in   76.22 millis
   usr time   49.00 millis
   sys time   24.59 millis

Signed-off-by: Torsten Rasmussen <Torsten.Rasmussen@nordicsemi.no>

CMakeLists.txt

@@ -880,7 +880,7 @@ if(CONFIG_USERSPACE)
     OUTPUT ${APP_SMEM_UNALIGNED_LD} ${APP_SMEM_PINNED_UNALIGNED_LD}
     COMMAND ${PYTHON_EXECUTABLE}
     ${ZEPHYR_BASE}/scripts/gen_app_partitions.py
-    -d ${OBJ_FILE_DIR}
+    -f ${CMAKE_BINARY_DIR}/compile_commands.json
     -o ${APP_SMEM_UNALIGNED_LD}
     $<$<BOOL:${APP_SMEM_PINNED_UNALIGNED_LD}>:--pinoutput=${APP_SMEM_PINNED_UNALIGNED_LD}>
     ${APP_SMEM_PINNED_PARTITION_LIST_ARG}

cmake/app/boilerplate.cmake

@@ -558,6 +558,14 @@ set(SOC_SERIES ${CONFIG_SOC_SERIES})
 set(SOC_TOOLCHAIN_NAME ${CONFIG_SOC_TOOLCHAIN_NAME})
 set(SOC_FAMILY ${CONFIG_SOC_FAMILY})
 
+# For the gen_app_partitions.py to work correctly, we must ensure that
+# all targets exports their compile commands to fetch object files.
+# We enable it unconditionally, as this is also useful for several IDEs
+set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE CACHE BOOL
+    "Export CMake compile commands. Used by gen_app_partitions.py script"
+    FORCE
+)
+
 if("${SOC_SERIES}" STREQUAL "")
   set(SOC_PATH ${SOC_NAME})
 else()

scripts/gen_app_partitions.py

@@ -34,6 +34,7 @@ found, into data and BSS for each partition.
 
 import sys
 import argparse
+import json
 import os
 import re
 from collections import OrderedDict
@@ -149,6 +150,26 @@ def parse_obj_files(partitions):
                     find_obj_file_partitions(fullname, partitions)
 
 
+def parse_compile_command_file(partitions):
+    # Iterate over all entries to find object files.
+    # Thereafter process each object file to find partitions
+    object_pattern = re.compile(r'-o\s+(\S*)')
+    with open(args.compile_commands_file, 'rb') as f:
+        commands = json.load(f)
+        for command in commands:
+            build_dir = command.get('directory')
+            compile_command = command.get('command')
+            compile_arg = object_pattern.search(compile_command)
+            obj_file = None if compile_arg is None else compile_arg.group(1)
+            if obj_file:
+                fullname = os.path.join(build_dir, obj_file)
+                # Because of issue #40635, then not all objects referenced by
+                # the compile_commands.json file may be available, therefore
+                # only include existing files.
+                if os.path.exists(fullname):
+                    find_obj_file_partitions(fullname, partitions)
+
+
 def parse_elf_file(partitions):
     with open(args.elf, 'rb') as f:
         try:
@@ -216,6 +237,8 @@ def parse_args():
                         help="Root build directory")
     parser.add_argument("-e", "--elf", required=False, default=None,
                         help="ELF file")
+    parser.add_argument("-f", "--compile-commands-file", required=False,
+                        default=None, help="CMake compile commands file")
     parser.add_argument("-o", "--output", required=False,
                         help="Output ld file")
     parser.add_argument("-v", "--verbose", action="count", default=0,
@@ -237,6 +260,8 @@ def main():
 
     if args.directory is not None:
         parse_obj_files(partitions)
+    if args.compile_commands_file is not None:
+        parse_compile_command_file(partitions)
     elif args.elf is not None:
         parse_elf_file(partitions)
     else: