x86: ia32/linker: add boot and pinned sections

This adds both boot and pinned sections to the linker
script for ia32. This is required for enabling demand
paging for kernel and data.

Signed-off-by: Daniel Leung <daniel.leung@intel.com>

arch/x86/core/ia32/crt0.S

@@ -251,6 +251,12 @@ __csSet:
 #endif /* Z_VM_KERNEL */
 
 	/* Clear BSS */
+#ifdef CONFIG_LINKER_USE_BOOT_SECTION
+	call	z_bss_zero_boot
+#endif
+#ifdef CONFIG_LINKER_USE_PINNED_SECTION
+	call	z_bss_zero_pinned
+#endif
 	call	z_bss_zero
 
 	/* load 32-bit operand size IDT */

include/arch/x86/ia32/linker.ld

@@ -87,6 +87,202 @@ SECTIONS
 	*(.iplt)
 	}
 
+#ifdef CONFIG_LINKER_USE_BOOT_SECTION
+
+	SECTION_PROLOGUE(boot.text,,)
+	{
+#include <snippets-rom-start.ld>
+
+		MMU_PAGE_ALIGN
+		lnkr_boot_start = .;
+		z_mapped_start = .;
+
+		lnkr_boot_text_start = .;
+
+		KEEP(*(.boot_text.__start))
+		*(.boot_text)
+		*(.boot_text.*)
+
+		MMU_PAGE_ALIGN_PERM
+
+		lnkr_boot_text_end = .;
+	} GROUP_ROM_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	SECTION_PROLOGUE(boot.rodata,,)
+	{
+		MMU_PAGE_ALIGN_PERM
+
+		lnkr_boot_rodata_start = .;
+
+		*(.boot_rodata)
+		*(.boot_rodata.*)
+
+		MMU_PAGE_ALIGN_PERM
+
+		lnkr_boot_rodata_end = .;
+	} GROUP_ROM_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	SECTION_PROLOGUE(boot.data,,)
+	{
+		MMU_PAGE_ALIGN_PERM
+
+		. = ALIGN(4);
+
+		lnkr_boot_data_start = .;
+
+		*(.boot_data)
+		*(.boot_data.*)
+
+		lnkr_boot_data_end = .;
+	} GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	SECTION_PROLOGUE(boot.bss, (NOLOAD),)
+	{
+		. = ALIGN(4);
+
+		lnkr_boot_bss_start = .;
+
+		*(.boot_bss)
+		*(.boot_bss.*)
+
+		lnkr_boot_bss_end = .;
+	} GROUP_NOLOAD_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	SECTION_PROLOGUE(boot.noinit, (NOLOAD),)
+	{
+		. = ALIGN(4);
+
+		lnkr_boot_noinit_start = .;
+
+		*(.boot_noinit)
+		*(.boot_noinit.*)
+
+		lnkr_boot_noinit_end = .;
+
+		MMU_PAGE_ALIGN
+
+		lnkr_boot_end = .;
+	} GROUP_NOLOAD_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	lnkr_boot_text_size = lnkr_boot_text_end - lnkr_boot_text_start;
+	lnkr_boot_rodata_size = lnkr_boot_rodata_end - lnkr_boot_rodata_start;
+	lnkr_boot_data_size = lnkr_boot_data_end - lnkr_boot_data_start;
+	lnkr_boot_bss_size = lnkr_boot_bss_end - lnkr_boot_bss_start;
+	lnkr_boot_noinit_size = lnkr_boot_noinit_end - lnkr_boot_noinit_start;
+
+#endif /* CONFIG_LINKER_USE_BOOT_SECTION */
+
+#ifdef CONFIG_LINKER_USE_PINNED_SECTION
+
+	SECTION_PROLOGUE(pinned.text,,)
+	{
+#ifndef CONFIG_LINKER_USE_BOOT_SECTION
+#include <snippets-rom-start.ld>
+#endif
+
+		MMU_PAGE_ALIGN
+
+		lnkr_pinned_start = .;
+
+#ifndef CONFIG_LINKER_USE_BOOT_SECTION
+		z_mapped_start = .;
+#endif
+
+		lnkr_pinned_text_start = .;
+
+		*(.pinned_text)
+		*(.pinned_text.*)
+
+		*(.gnu.linkonce.t.exc_*)
+
+#include <linker/kobject-text.ld>
+
+		MMU_PAGE_ALIGN_PERM
+
+		lnkr_pinned_text_end = .;
+	} GROUP_ROM_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	MMU_PAGE_ALIGN_PERM
+
+	lnkr_pinned_rodata_start = .;
+
+#include <linker/common-rom.ld>
+#include <linker/thread-local-storage.ld>
+
+	SECTION_PROLOGUE(pinned.rodata,,)
+	{
+#include <arch/x86/ia32/scripts/static_intr.ld>
+
+		*(.pinned_rodata)
+		*(.pinned_rodata.*)
+
+#include <snippets-rodata.ld>
+
+#include <linker/kobject-rom.ld>
+
+		MMU_PAGE_ALIGN_PERM
+
+		lnkr_pinned_rodata_end = .;
+	} GROUP_ROM_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	SECTION_PROLOGUE(pinned.data,,)
+	{
+		MMU_PAGE_ALIGN_PERM
+
+		lnkr_pinned_data_start = .;
+
+		. = ALIGN(4);
+
+#include <arch/x86/ia32/scripts/shared_kernel_pages.ld>
+#include <arch/x86/ia32/scripts/dynamic_intr.ld>
+
+		*(.pinned_data)
+		*(.pinned_data.*)
+
+	} GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+#include <linker/common-ram.ld>
+#include <arch/x86/pagetables.ld>
+#include <linker/kobject-data.ld>
+
+	lnkr_pinned_data_end = .;
+
+	SECTION_PROLOGUE(pinned.bss, (NOLOAD),)
+	{
+		. = ALIGN(4);
+
+		lnkr_pinned_bss_start = .;
+
+		*(.pinned_bss)
+		*(.pinned_bss.*)
+
+		lnkr_pinned_bss_end = .;
+	} GROUP_NOLOAD_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	SECTION_PROLOGUE(pinned.noinit, (NOLOAD),)
+	{
+		. = ALIGN(4);
+
+		lnkr_pinned_noinit_start = .;
+
+		*(.pinned_noinit)
+		*(.pinned_noinit.*)
+
+		lnkr_pinned_noinit_end = .;
+
+		MMU_PAGE_ALIGN
+
+		lnkr_pinned_end = .;
+
+	} GROUP_NOLOAD_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
+
+	lnkr_pinned_text_size = lnkr_pinned_text_end - lnkr_pinned_text_start;
+	lnkr_pinned_rodata_size = lnkr_pinned_rodata_end - lnkr_pinned_rodata_start;
+	lnkr_pinned_data_size = lnkr_pinned_data_end - lnkr_pinned_data_start;
+	lnkr_pinned_bss_size = lnkr_pinned_bss_end - lnkr_pinned_bss_start;
+	lnkr_pinned_noinit_size = lnkr_pinned_noinit_end - lnkr_pinned_noinit_start;
+
+#endif /* CONFIG_LINKER_USE_PINNED_SECTION */
 
 	GROUP_START(ROMABLE_REGION)
 
@@ -98,15 +294,22 @@ SECTIONS
 	SECTION_PROLOGUE(_TEXT_SECTION_NAME,,)
 	{
 	_image_text_start = .;
+
+#if !defined(CONFIG_LINKER_USE_BOOT_SECTION) || \
+    !defined(CONFIG_LINKER_USE_PINNED_SECTION)
 #ifndef CONFIG_XIP
 	z_mapped_start = .;
 #endif
+#endif
 
+#if !defined(CONFIG_LINKER_USE_BOOT_SECTION) || \
+    !defined(CONFIG_LINKER_USE_PINNED_SECTION)
 /* Located in generated directory. This file is populated by calling
  * zephyr_linker_sources(ROM_START ...). This typically contains the vector
  * table and debug information.
  */
 #include <snippets-rom-start.ld>
+#endif
 
 	/* Needs KEEP() as ENTRY() is given a physical address */
 	KEEP(*(.text.__start))
@@ -119,7 +322,9 @@ SECTIONS
 	*(.fini)
 	*(.eini)
 
+#ifndef CONFIG_LINKER_USE_PINNED_SECTION
 #include <linker/kobject-text.ld>
+#endif
 
 	MMU_PAGE_ALIGN_PERM
 	} GROUP_ROM_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
@@ -128,8 +333,10 @@ SECTIONS
 	_image_text_size = _image_text_end - _image_text_start;
 	_image_rodata_start = .;
 
+#ifndef CONFIG_LINKER_USE_PINNED_SECTION
 #include <linker/common-rom.ld>
 #include <linker/thread-local-storage.ld>
+#endif
 
 	SECTION_PROLOGUE(_RODATA_SECTION_NAME,,)
 	{
@@ -138,29 +345,19 @@ SECTIONS
 	*(.gnu.linkonce.r.*)
 
 #ifndef CONFIG_DYNAMIC_INTERRUPTS
-	. = ALIGN(8);
-	_idt_base_address = .;
-#ifdef LINKER_ZEPHYR_FINAL
-	KEEP(*(staticIdt))
-#else
-	. += CONFIG_IDT_NUM_VECTORS * 8;
-#endif /* LINKER_ZEPHYR_FINAL */
-
-	. = ALIGN(4);
-	_irq_to_interrupt_vector = .;
-#ifdef LINKER_ZEPHYR_FINAL
-	KEEP(*(irq_int_vector_map))
-#else
-	. += CONFIG_MAX_IRQ_LINES;
-#endif
+#ifndef CONFIG_LINKER_USE_PINNED_SECTION
+#include <arch/x86/ia32/scripts/static_intr.ld>
+#endif /* !CONFIG_LINKER_USE_PINNED_SECTION */
 #endif /* CONFIG_DYNAMIC_INTERRUPTS */
 
+#ifndef CONFIG_LINKER_USE_PINNED_SECTION
 /* Located in generated directory. This file is populated by the
  * zephyr_linker_sources() Cmake function.
  */
 #include <snippets-rodata.ld>
 
 #include <linker/kobject-rom.ld>
+#endif
 	} GROUP_ROM_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
 
 #include <linker/cplusplus-rom.ld>
@@ -252,20 +449,9 @@ SECTIONS
 	*(".kernel.*")
 
 #ifdef CONFIG_DYNAMIC_INTERRUPTS
-	. = ALIGN(4);
-	_irq_to_interrupt_vector = .;
-#ifdef LINKER_ZEPHYR_FINAL
-	KEEP(*(irq_int_vector_map))
-#else
-	. += CONFIG_MAX_IRQ_LINES;
-#endif /* LINKER_ZEPHYR_FINAL */
-
-	z_interrupt_vectors_allocated = .;
-#ifdef LINKER_ZEPHYR_FINAL
-	KEEP(*(irq_vectors_alloc))
-#else
-	. += (CONFIG_IDT_NUM_VECTORS + 7) / 8;
-#endif /* LINKER_ZEPHYR_FINAL */
+#ifndef CONFIG_LINKER_USE_PINNED_SECTION
+#include <arch/x86/ia32/scripts/dynamic_intr.ld>
+#endif /* !CONFIG_LINKER_USE_PINNED_SECTION */
 #endif /* CONFIG_DYNAMIC_INTERRUPTS */
 
 /* Located in generated directory. This file is populated by the
@@ -273,69 +459,24 @@ SECTIONS
  */
 #include <snippets-rwdata.ld>
 
-#ifdef CONFIG_X86_KPTI
-	MMU_PAGE_ALIGN_PERM
-	z_shared_kernel_page_start = .;
-	/* Special page containing supervisor data that is still mapped in
-	 * user mode page tables. IDT, GDT, TSSes, trampoline stack, and
-	 * any LDT must go here as they always must live in a page that is
-	 * marked 'present'. Still not directly user accessible, but
-	 * no sensitive data should be here as Meltdown exploits may read it.
-	 */
-#endif /* CONFIG_X86_KPTI */
-
-#ifdef CONFIG_DYNAMIC_INTERRUPTS
-	. = ALIGN(8);
-	_idt_base_address = .;
-#ifdef LINKER_ZEPHYR_FINAL
-	KEEP(*(staticIdt))
-#else
-	. += CONFIG_IDT_NUM_VECTORS * 8;
-#endif /* LINKER_ZEPHYR_FINAL */
-#endif /* CONFIG_DYNAMIC_INTERRUPTS */
-
-#ifdef CONFIG_GDT_DYNAMIC
-	KEEP(*(.tss))
-	. = ALIGN(8);
-	_gdt = .;
-#ifdef LINKER_ZEPHYR_FINAL
-	KEEP(*(gdt))
-#else /* LINKER_ZEPHYR_FINAL */
-
-#ifdef CONFIG_USERSPACE
-    #define GDT_NUM_ENTRIES 7
-#elif defined(CONFIG_HW_STACK_PROTECTION)
-    #define GDT_NUM_ENTRIES 5
-#else
-    #define GDT_NUM_ENTRIES 3
-#endif /* CONFIG_X86_USERSPACE */
-	. += GDT_NUM_ENTRIES * 8;
-#endif /* LINKER_ZEPHYR_FINAL */
-#endif /* CONFIG_GDT_DYNAMIC */
-
-#ifdef CONFIG_X86_KPTI
-	z_trampoline_stack_start = .;
-	MMU_PAGE_ALIGN_PERM
-	z_trampoline_stack_end = .;
-	z_shared_kernel_page_end = .;
-
-	ASSERT(z_trampoline_stack_end - z_trampoline_stack_start >= 40,
-		"trampoline stack too small");
-	ASSERT(z_shared_kernel_page_end - z_shared_kernel_page_start == 4096,
-	       "shared kernel area is not one memory page");
-#endif /* CONFIG_X86_KPTI */
+#ifndef CONFIG_LINKER_USE_PINNED_SECTION
+#include <arch/x86/ia32/scripts/shared_kernel_pages.ld>
+#endif /* !CONFIG_LINKER_USE_PINNED_SECTION */
 
 	. = ALIGN(4);
 	} GROUP_DATA_LINK_IN(RAMABLE_REGION, ROMABLE_REGION)
 
 	__data_rom_start = LOADADDR(_DATA_SECTION_NAME);
 
-#include <linker/common-ram.ld>
 #include <linker/cplusplus-ram.ld>
+
+#ifndef CONFIG_LINKER_USE_PINNED_SECTION
+#include <linker/common-ram.ld>
 #include <arch/x86/pagetables.ld>
 
 /* Must be last in RAM */
 #include <linker/kobject-data.ld>
+#endif /* !CONFIG_LINKER_USE_PINNED_SECTION */
 
 	MMU_PAGE_ALIGN
 	__data_ram_end = .;

include/arch/x86/ia32/scripts/dynamic_intr.ld

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifdef CONFIG_DYNAMIC_INTERRUPTS
+	. = ALIGN(4);
+	_irq_to_interrupt_vector = .;
+#ifdef LINKER_ZEPHYR_FINAL
+	KEEP(*(irq_int_vector_map))
+#else
+	. += CONFIG_MAX_IRQ_LINES;
+#endif /* LINKER_ZEPHYR_FINAL */
+
+	z_interrupt_vectors_allocated = .;
+#ifdef LINKER_ZEPHYR_FINAL
+	KEEP(*(irq_vectors_alloc))
+#else
+	. += (CONFIG_IDT_NUM_VECTORS + 7) / 8;
+#endif /* LINKER_ZEPHYR_FINAL */
+#endif /* CONFIG_DYNAMIC_INTERRUPTS */

include/arch/x86/ia32/scripts/shared_kernel_pages.ld

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifdef CONFIG_X86_KPTI
+	MMU_PAGE_ALIGN_PERM
+	z_shared_kernel_page_start = .;
+	/* Special page containing supervisor data that is still mapped in
+	 * user mode page tables. IDT, GDT, TSSes, trampoline stack, and
+	 * any LDT must go here as they always must live in a page that is
+	 * marked 'present'. Still not directly user accessible, but
+	 * no sensitive data should be here as Meltdown exploits may read it.
+	 */
+#endif /* CONFIG_X86_KPTI */
+
+#ifdef CONFIG_DYNAMIC_INTERRUPTS
+	. = ALIGN(8);
+	_idt_base_address = .;
+#ifdef LINKER_ZEPHYR_FINAL
+	KEEP(*(staticIdt))
+#else
+	. += CONFIG_IDT_NUM_VECTORS * 8;
+#endif /* LINKER_ZEPHYR_FINAL */
+#endif /* CONFIG_DYNAMIC_INTERRUPTS */
+
+#ifdef CONFIG_GDT_DYNAMIC
+	KEEP(*(.tss))
+	. = ALIGN(8);
+	_gdt = .;
+#ifdef LINKER_ZEPHYR_FINAL
+	KEEP(*(gdt))
+#else /* LINKER_ZEPHYR_FINAL */
+
+#ifdef CONFIG_USERSPACE
+    #define GDT_NUM_ENTRIES 7
+#elif defined(CONFIG_HW_STACK_PROTECTION)
+    #define GDT_NUM_ENTRIES 5
+#else
+    #define GDT_NUM_ENTRIES 3
+#endif /* CONFIG_X86_USERSPACE */
+	. += GDT_NUM_ENTRIES * 8;
+#endif /* LINKER_ZEPHYR_FINAL */
+#endif /* CONFIG_GDT_DYNAMIC */
+
+#ifdef CONFIG_X86_KPTI
+	z_trampoline_stack_start = .;
+	MMU_PAGE_ALIGN_PERM
+	z_trampoline_stack_end = .;
+	z_shared_kernel_page_end = .;
+
+	ASSERT(z_trampoline_stack_end - z_trampoline_stack_start >= 40,
+		"trampoline stack too small");
+	ASSERT(z_shared_kernel_page_end - z_shared_kernel_page_start == 4096,
+	       "shared kernel area is not one memory page");
+#endif /* CONFIG_X86_KPTI */

include/arch/x86/ia32/scripts/static_intr.ld

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2021 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef CONFIG_DYNAMIC_INTERRUPTS
+	. = ALIGN(8);
+	_idt_base_address = .;
+#ifdef LINKER_ZEPHYR_FINAL
+	KEEP(*(staticIdt))
+#else
+	. += CONFIG_IDT_NUM_VECTORS * 8;
+#endif /* LINKER_ZEPHYR_FINAL */
+
+	. = ALIGN(4);
+	_irq_to_interrupt_vector = .;
+#ifdef LINKER_ZEPHYR_FINAL
+	KEEP(*(irq_int_vector_map))
+#else
+	. += CONFIG_MAX_IRQ_LINES;
+#endif
+#endif /* CONFIG_DYNAMIC_INTERRUPTS */