build: simplfy how extra build steps are specified

For various reasons its often necessary to generate certain
complex data structures at build-time by separate tools outside
of the C compiler. Data is populated to these tools by way of
special binary sections not intended to be included in the final
binary. We currently do this to generate interrupt tables, forthcoming
work will also use this to generate MMU page tables.

The way we have been doing this is to generatea "kernel_prebuilt.elf",
extract the metadata sections with objcopy, run the tool, and then
re-link the kernel with the extra data *and* use objcopy to pull
out the unwanted sections.

This doesn't scale well if multiple post-build steps are needed.
Now this is much simpler; in any Makefile, a special
GENERATED_KERNEL_OBJECT_FILES variable may be appended to containing
the filenames to the generated object files, which will be generated
by Make in the usual fashion.

Instead of using objcopy to pull out, we now create a linker-pass2.cmd
which additionally defines LINKER_PASS2. The source linker script
can #ifdef around this to use the special /DISCARD/ section target
to not include metadata sections in the final binary.

Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>

Makefile

@@ -364,6 +364,7 @@ KERNEL_ELF_NAME = $(KERNEL_NAME).elf
 KERNEL_BIN_NAME = $(KERNEL_NAME).bin
 KERNEL_HEX_NAME = $(KERNEL_NAME).hex
 KERNEL_STAT_NAME = $(KERNEL_NAME).stat
+PREBUILT_KERNEL = $(KERNEL_NAME)_prebuilt.elf
 
 export SOC_FAMILY SOC_SERIES SOC_PATH SOC_NAME BOARD_NAME
 export ARCH KERNEL_NAME KERNEL_ELF_NAME KERNEL_BIN_NAME KERNEL_HEX_NAME
@@ -852,13 +853,14 @@ $(KERNEL_NAME).lnk: $(zephyr-deps)
 
 linker.cmd: $(zephyr-deps)
 	$(Q)$(CC) -x assembler-with-cpp -nostdinc -undef -E -P \
-	$(LDFLAG_LINKERCMD) $(LD_TOOLCHAIN) -I$(srctree)/include \
-	-I$(SOURCE_DIR) \
-	-I$(objtree)/include/generated $(EXTRA_LINKER_CMD_OPT) $(KBUILD_LDS) -o $@
+		$(LDFLAG_LINKERCMD) $(LD_TOOLCHAIN) \
+		-I$(srctree)/include -I$(SOURCE_DIR) \
+		-I$(objtree)/include/generated \
+		$(EXTRA_LINKER_CMD_OPT) $(KBUILD_LDS) -o $@
 
-PREBUILT_KERNEL = $(KERNEL_NAME)_prebuilt.elf
 
-$(PREBUILT_KERNEL): $(zephyr-deps) libzephyr.a $(KBUILD_ZEPHYR_APP) $(app-y) linker.cmd $(KERNEL_NAME).lnk
+$(PREBUILT_KERNEL): $(zephyr-deps) libzephyr.a $(KBUILD_ZEPHYR_APP) $(app-y) \
+		linker.cmd $(KERNEL_NAME).lnk
 	$(Q)$(CC) -T linker.cmd @$(KERNEL_NAME).lnk -o $@
 
 ASSERT_WARNING_STR := \
@@ -878,15 +880,39 @@ WARN_ABOUT_DEPRECATION := $(if $(CONFIG_BOARD_DEPRECATED),echo -e \
 				-n $(DEPRECATION_WARNING_STR),true)
 
 ifeq ($(ARCH),x86)
-# X86 with its IDT has very special handling for interrupt tables
 include $(srctree)/arch/x86/Makefile.idt
-else ifeq ($(CONFIG_GEN_ISR_TABLES),y)
-# Logic for interrupt tables created by scripts/gen_isr_tables.py
+endif
+
+ifeq ($(CONFIG_GEN_ISR_TABLES),y)
 include $(srctree)/arch/common/Makefile.gen_isr_tables
-else
+endif
+
+ifneq ($(GENERATED_KERNEL_OBJECT_FILES),)
+
+# Identical rule to linker.cmd, but we also define preprocessor LINKER_PASS2.
+# For arches that place special metadata in $(PREBUILT_KERNEL) not intended
+# for the final binary, it can be #ifndef'd around this.
+linker-pass2.cmd: $(zephyr-deps)
+	$(Q)$(CC) -x assembler-with-cpp -nostdinc -undef -E -P \
+		-DLINKER_PASS2 \
+		$(LDFLAG_LINKERCMD) $(LD_TOOLCHAIN) \
+		-I$(srctree)/include -I$(SOURCE_DIR) \
+		-I$(objtree)/include/generated \
+		$(EXTRA_LINKER_CMD_OPT) $(KBUILD_LDS) -o $@
+
+$(KERNEL_ELF_NAME): $(GENERATED_KERNEL_OBJECT_FILES) linker-pass2.cmd
+	$(Q)$(CC) -T linker-pass2.cmd $(GENERATED_KERNEL_OBJECT_FILES) \
+		@$(KERNEL_NAME).lnk -o $@
+	$(Q)$(srctree)/scripts/check_link_map.py $(KERNEL_NAME).map
+	@$(WARN_ABOUT_ASSERT)
+	@$(WARN_ABOUT_DEPRECATION)
+
+else # GENERATED_KERNEL_OBJECT_FILES
+
 # Otherwise, nothing to do, prebuilt kernel is the real one
 $(KERNEL_ELF_NAME): $(PREBUILT_KERNEL)
-	@cp $(PREBUILT_KERNEL) $(KERNEL_ELF_NAME)
+	$(Q)cp $(PREBUILT_KERNEL) $(KERNEL_ELF_NAME)
+	$(Q)$(srctree)/scripts/check_link_map.py $(KERNEL_NAME).map
 	@$(WARN_ABOUT_ASSERT)
 	@$(WARN_ABOUT_DEPRECATION)
 endif
@@ -1066,7 +1092,8 @@ CLEAN_DIRS  += $(MODVERDIR)
 CLEAN_FILES += 	include/generated/generated_dts_board.h \
 		.old_version .tmp_System.map .tmp_version \
 		.tmp_* System.map *.lnk *.map *.elf *.lst \
-		*.bin *.hex *.stat *.strip staticIdt.o linker.cmd
+		*.bin *.hex *.stat *.strip staticIdt.o linker.cmd \
+		linker-pass2.cmd
 
 # Directories & files removed with 'make mrproper'
 MRPROPER_DIRS  += bin include/config usr/include include/generated          \

arch/common/Makefile.gen_isr_tables

@@ -37,31 +37,4 @@ quiet_cmd_gen_irq = IRQ     $@
 $(OUTPUT_SRC): $(PREBUILT_KERNEL) $(GEN_ISR_TABLE)
 	$(call cmd,gen_irq)
 
-# Build system pattern rules will handle building $(OUTPUT_OBJ) from
-# $(OUTPUT_SRC), nothing we need to do here explicitly for its compilation.
-
-# Now link the kernel again, this time with the compiled interrupt tables
-# included, replacing the dummy tables defined in arch/common/isr_tables.c
-#
-# On x86, we just strip out the intList with objcopy -j. However this is not
-# very portable; for instance on ARM this results in a zero-sized program
-# header segment which produces a linker warning and gives QEMU fits.
-# Set to NOLOAD instead, now that we have extracted the information we need
-# from it.
-quiet_cmd_lnk_elf = LINK    $@
-      cmd_lnk_elf = \
-( \
-	$(CC) -T linker.cmd $(OUTPUT_OBJ) @$(KERNEL_NAME).lnk \
-		-o elf.tmp && \
-	$(OBJCOPY) -I $(OUTPUT_FORMAT) -O $(OUTPUT_FORMAT) \
-		--set-section-flags .intList=noload \
-		elf.tmp $@ && \
-	rm -f elf.tmp; \
-)
-
-$(KERNEL_ELF_NAME): $(OUTPUT_OBJ) linker.cmd
-	$(call cmd,lnk_elf)
-	@$(srctree)/scripts/check_link_map.py $(KERNEL_NAME).map
-	@$(WARN_ABOUT_ASSERT)
-	@$(WARN_ABOUT_DEPRECATION)
-
+GENERATED_KERNEL_OBJECT_FILES += $(OUTPUT_OBJ)

arch/x86/Makefile.idt

@@ -34,20 +34,7 @@ $(GENIDT):
 staticIdt.o: $(PREBUILT_KERNEL) $(GENIDT)
 	$(call cmd,gen_idt)
 
-quiet_cmd_lnk_elf = LINK    $@
-      cmd_lnk_elf =								\
-(										\
-	$(CC) -T linker.cmd @$(KERNEL_NAME).lnk staticIdt.o			\
-		irq_int_vector_map.o -o $@;					\
-	${OBJCOPY} --change-section-address intList=${CONFIG_PHYS_LOAD_ADDR}	\
-		$@ elf.tmp;							\
-	$(OBJCOPY) -R intList elf.tmp $@;					\
-	rm elf.tmp								\
-)
+irq_int_vector_map.o: staticIdt.o
 
-$(KERNEL_ELF_NAME): staticIdt.o linker.cmd
-	$(call cmd,lnk_elf)
-	@$(srctree)/scripts/check_link_map.py $(KERNEL_NAME).map
-	@$(WARN_ABOUT_ASSERT)
-	@$(WARN_ABOUT_DEPRECATION)
+GENERATED_KERNEL_OBJECT_FILES += staticIdt.o irq_int_vector_map.o
 

include/arch/x86/linker.ld

@@ -194,6 +194,7 @@ SECTIONS
 
 	GROUP_END(RAMABLE_REGION)
 
+#ifndef LINKER_PASS2
 	/* static interrupts */
 	SECTION_PROLOGUE(intList, (OPTIONAL),)
 	{
@@ -205,6 +206,15 @@ SECTIONS
 	KEEP(*(.gnu.linkonce.intList.*))
 	__INT_LIST_END__ = .;
 	} > IDT_LIST
+#else
+	/DISCARD/ :
+	{
+	KEEP(*(.spurIsr))
+	KEEP(*(.spurNoErrIsr))
+	KEEP(*(.intList))
+	KEEP(*(.gnu.linkonce.intList.*))
+	}
+#endif
 
 #ifdef CONFIG_CUSTOM_SECTIONS_LD
 /* Located in project source directory */

include/linker/intlist.ld

@@ -34,12 +34,7 @@
  * }
  */
 
-/* We don't set NOLOAD here, as objcopy will not let us extract the intList
- * section into a file if we do so; the resulting file is 0 bytes. We do so
- * with objcopy in Makefile.gen_isr_tables after we have extracted the
- * information we need.
- */
-
+#ifndef LINKER_PASS2
 SECTION_PROLOGUE(.intList,,)
 {
 	KEEP(*(.irq_info))
@@ -48,4 +43,10 @@ SECTION_PROLOGUE(.intList,,)
 	KEEP(*(.intList))
 	__INT_LIST_END__ = .;
 } GROUP_LINK_IN(IDT_LIST)
-
+#else
+/DISCARD/ :
+{
+	KEEP(*(.irq_info))
+	KEEP(*(.intList))
+}
+#endif